From fb366cc0914e386a0069e6aeb9b3ce906a092a88 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Fri, 9 Oct 2026 11:11:30 +0800 Subject: [PATCH 01/19] fix(calvin): rebuild scheduler state from the sequencer log on restart A Calvin scheduler now records the highest epoch it applied and, after a restart, re-applies sequencer log entries up to that epoch before the node opens its client gateway. Startup waits on every local scheduler and fails when the data-group recovery bound expires or a scheduler halts. A newly elected sequencer leader seeds its epoch from the installed sequencer snapshot instead of refusing to propose on a compacted log. Related changes: - Completion routing, waiters, verdict handling and the owed-completion retry path track per-transaction history in the sequencer state machine and surface typed abort errors to the planner retry loop. - Scheduler channel capacity, transaction deadline, passive-read timeout, verdict stall warning, in-flight backlog and catch-up window are configurable under `tuning.calvin`. - The bridge dispatcher's per-tenant in-flight accounting moves into its own type. - Raft leaders track voter contact from the election win, so a voter not yet heard this term counts from that point. - The CI nextest profile runs every test, including the cluster test that was excluded. --- .config/nextest.toml | 14 +- docs/getting-started.md | 31 + nodedb-cluster/src/calvin/completion.rs | 633 ++++++++---------- nodedb-cluster/src/calvin/completion_entry.rs | 62 +- nodedb-cluster/src/calvin/completion_gc.rs | 112 +++- nodedb-cluster/src/calvin/completion_parts.rs | 38 +- .../src/calvin/completion_verdict.rs | 62 +- .../src/calvin/completion_waiter.rs | 9 + nodedb-cluster/src/calvin/mod.rs | 4 +- nodedb-cluster/src/calvin/sequencer/entry.rs | 49 +- nodedb-cluster/src/calvin/sequencer/mod.rs | 2 +- nodedb-cluster/src/calvin/sequencer/replay.rs | 2 - .../src/calvin/sequencer/service/core.rs | 33 +- .../calvin/sequencer/service/epoch_mint.rs | 4 +- .../calvin/sequencer/service/epoch_seed.rs | 217 +++++- .../calvin/sequencer/state_machine/apply.rs | 45 +- .../calvin/sequencer/state_machine/core.rs | 3 + .../calvin/sequencer/state_machine/history.rs | 65 ++ .../src/calvin/sequencer/state_machine/mod.rs | 2 + .../sequencer/state_machine/snapshot.rs | 36 +- nodedb-cluster/src/multi_raft/proposals.rs | 8 +- .../src/raft_loop/handle_rpc/consensus.rs | 7 +- .../src/raft_loop/tick/apply_committed.rs | 30 +- nodedb-raft/src/node/core.rs | 4 + nodedb-raft/src/node/internal.rs | 1 + nodedb-raft/src/node/peer_contact.rs | 139 +++- nodedb-raft/src/node/quorum_contact.rs | 14 + nodedb-types/src/config/tuning/calvin.rs | 120 ++++ nodedb-types/src/config/tuning/config.rs | 15 + nodedb-types/src/config/tuning/mod.rs | 2 + nodedb/src/bootstrap/calvin_catch_up.rs | 181 +++++ nodedb/src/bootstrap/cluster_ready.rs | 9 + nodedb/src/bootstrap/mod.rs | 1 + nodedb/src/bridge/dispatch/core_channel.rs | 2 +- nodedb/src/bridge/dispatch/dispatcher.rs | 18 +- nodedb/src/bridge/dispatch/drain.rs | 4 +- nodedb/src/bridge/dispatch/enqueue.rs | 132 ++-- nodedb/src/bridge/dispatch/mod.rs | 1 + nodedb/src/bridge/dispatch/response_poll.rs | 38 +- nodedb/src/bridge/dispatch/tenant_inflight.rs | 189 ++++++ nodedb/src/config/server/config.rs | 39 ++ nodedb/src/config/server/domain.rs | 33 + .../calvin/scheduler/applied_mirror.rs | 11 + .../cluster/calvin/scheduler/caught_up.rs | 127 ++++ .../cluster/calvin/scheduler/driver/config.rs | 101 ++- .../scheduler/driver/core/commit_redo.rs | 2 +- .../driver/core/commit_resolve/apply_tail.rs | 67 +- .../driver/core/commit_resolve/verdict.rs | 27 +- .../driver/core/commit_resolve/vote.rs | 9 +- .../scheduler/driver/core/completion_route.rs | 154 +---- .../calvin/scheduler/driver/core/deferred.rs | 28 +- .../driver/core/dispatch/active_dispatch.rs | 39 +- .../driver/core/dispatch/incarnation.rs | 10 +- .../scheduler/driver/core/dispatch/mod.rs | 1 + .../driver/core/dispatch/static_dispatch.rs | 57 +- .../driver/core/dispatch/unstaged.rs | 285 ++++++++ .../scheduler/driver/core/flush_turn.rs | 28 +- .../calvin/scheduler/driver/core/halt.rs | 30 +- .../calvin/scheduler/driver/core/mod.rs | 1 + .../scheduler/driver/core/owed/entry.rs | 64 +- .../scheduler/driver/core/owed/retry.rs | 58 +- .../calvin/scheduler/driver/core/parts.rs | 236 +++++-- .../scheduler/driver/core/parts_error.rs | 44 ++ .../calvin/scheduler/driver/core/process.rs | 10 +- .../scheduler/driver/core/read_result.rs | 112 +++- .../calvin/scheduler/driver/core/scheduler.rs | 89 ++- .../scheduler/driver/core/test_support.rs | 4 +- .../driver/core/write_version_record.rs | 231 +++++-- .../cluster/calvin/scheduler/driver/types.rs | 15 +- .../cluster/calvin/scheduler/metrics.rs | 8 +- .../cluster/calvin/scheduler/metrics_flow.rs | 3 - .../control/cluster/calvin/scheduler/mod.rs | 2 + .../src/control/cluster/sequencer_snapshot.rs | 42 +- .../src/control/cluster/start_raft/hooks.rs | 3 +- .../control/cluster/start_raft/loop_build.rs | 3 +- .../src/control/planner/calvin/abort_error.rs | 16 + .../control/planner/calvin/dependent_recon.rs | 6 +- .../src/control/planner/calvin/retry_loop.rs | 116 ++-- .../control/planner/calvin/submit/local.rs | 33 +- .../pgwire/handler/transaction_cmds/errors.rs | 20 - 80 files changed, 3118 insertions(+), 1384 deletions(-) create mode 100644 nodedb-cluster/src/calvin/sequencer/state_machine/history.rs create mode 100644 nodedb-types/src/config/tuning/calvin.rs create mode 100644 nodedb/src/bootstrap/calvin_catch_up.rs create mode 100644 nodedb/src/bridge/dispatch/tenant_inflight.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/caught_up.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/parts_error.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 26689fbb1..a24eff7de 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -212,18 +212,8 @@ server-process-serial = { max-threads = 1 } retries = { backoff = "fixed", count = 1, delay = "2s" } fail-fast = false -# One cluster test is excluded from the CI profile ONLY. It still runs under the -# default profile, which is what a local `cargo nextest run` uses. -# -# `learner_caught_up_via_real_install_snapshot` fails on CI runners about once in -# six runs. A newly elected sequencer leader whose log was compacted refuses to -# propose ("sequencer log was truncated below every retained epoch"), so the -# sequencer stalls. The joining node then never catches up on the sequencer -# group, and its authorization lease is withheld. Re-enable it once a leader can -# seed its epoch from the installed sequencer snapshot. NEVER widen a timeout. -default-filter = """ -not test(learner_caught_up_via_real_install_snapshot) -""" +# CI runs every test, including the two binaries the default profile excludes. +default-filter = "all()" # CI-only cluster headroom. On CI runners (~2× slower per-core than a dev # workstation) the cluster tests' in-test `wait_for` budgets are proportionally diff --git a/docs/getting-started.md b/docs/getting-started.md index 2ef35d74d..88306b15a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -589,6 +589,37 @@ a crashed node's descriptor leases block DDL until they expire. All three must be positive. `scope_expiry_interval_secs` has a floor of `10`. Below that the sweep costs more than the resolution it buys. +**Calvin scheduler settings** (config file only): + +| Config field | Default | +| ------------------------------------------------ | ------- | +| `tuning.calvin.channel_capacity` | `512` | +| `tuning.calvin.txn_deadline_multiplier` | `3` | +| `tuning.calvin.dependent_read_passive_timeout_ms` | `60` | +| `tuning.calvin.verdict_stall_warn_ms` | `5000` | +| `tuning.calvin.max_inflight_backlog` | `1024` | +| `tuning.calvin.catch_up_window` | `512` | + +Each hosted vShard runs one Calvin scheduler with these settings. + +- `channel_capacity` sizes each scheduler's bounded input and completion + channels. +- `txn_deadline_multiplier` sets a transaction deadline in sequencer epochs. +- `dependent_read_passive_timeout_ms` bounds the wait for passive read results. +- `verdict_stall_warn_ms` spaces the stall warnings of a transaction waiting + on its global verdict. The transaction keeps waiting and never aborts. +- `max_inflight_backlog` is the in-flight backlog at which a scheduler stops + taking new input. +- `catch_up_window` is the most sequencer log entries one catch-up pass + replays. + +Every value must be positive. `verdict_stall_warn_ms` has a floor of `4`. + +After a restart, a scheduler re-applies epochs up to the highest epoch it +applied before. The client gateway opens only after every scheduler on the +node reaches that epoch. The wait shares the data-group recovery bound, and +startup fails when the bound expires. + **Observability settings:** | Config field | Environment variable | Default | diff --git a/nodedb-cluster/src/calvin/completion.rs b/nodedb-cluster/src/calvin/completion.rs index a6e24fa46..af3b6f708 100644 --- a/nodedb-cluster/src/calvin/completion.rs +++ b/nodedb-cluster/src/calvin/completion.rs @@ -6,7 +6,7 @@ use std::time::{Duration, Instant}; use tokio::sync::{mpsc, oneshot}; -use crate::calvin::completion_entry::{PendingCompletion, outcome_for}; +use crate::calvin::completion_entry::PendingCompletion; use crate::calvin::completion_waiter::{CompletionReport, CompletionWaiter}; use crate::calvin::sequencer::AbortReason; @@ -37,25 +37,15 @@ impl TxnId { /// Terminal outcome of a single Calvin transaction attempt. /// -/// Exactly one of these fires per attempt on the unified completion channel: -/// all expected vshards acked AND the global verdict was commit (`Completed`), -/// all expected vshards acked but the global cross-shard verdict was ABORT -/// (`Aborted`), the executor reported an OLLP prediction mismatch that forces a -/// retry (`Mismatch`), or the scheduler rejected the transaction's routing as -/// terminally broken (`Failed`). `Failed` is never retried. `Aborted` retries -/// only for `AbortReason::PredictionDrift`, which is the same drift as -/// `Mismatch`. +/// Exactly one fires per attempt, once the global verdict is stored and every +/// expected vShard acked. A transaction that lost its parts fires at its +/// verdict, since no participant staged it. #[derive(Clone, Debug, PartialEq, Eq)] pub enum AttemptOutcome { + /// The global verdict was COMMIT. Completed, - /// The global cross-shard verdict was ABORT. `reason` names the cause. - Aborted { - reason: AbortReason, - }, - Mismatch, - Failed { - detail: String, - }, + /// The global verdict was ABORT. `reason` names the cause. + Aborted { reason: AbortReason }, } /// One participant vShard's durable commit vote. @@ -93,10 +83,6 @@ pub struct ParticipantProgress { pub acked: bool, /// The txn's global verdict is stored. pub has_verdict: bool, - /// An `OllpMismatch` for the txn is recorded. - pub mismatched: bool, - /// A `TxnRoutingFailed` for the txn is recorded and not yet delivered. - pub routing_failed: bool, } /// `pub(crate)`: also read by the vote/verdict-tally methods in @@ -115,32 +101,47 @@ pub(crate) struct Inner { /// Terminal entries with no waiter, oldest first, with the instant each /// became one (`completion_gc`). pub(crate) waiterless: VecDeque<(Instant, TxnId)>, - /// How long a terminal entry waits for a waiter before eviction. `None` - /// takes [`super::completion_gc::DEFAULT_WAITERLESS_TTL`]. + /// Entries no `EpochBatch` apply sequenced, oldest first, with the + /// instant each was created (`completion_gc`). + pub(crate) orphans: VecDeque<(Instant, TxnId)>, + /// How long a terminal entry waits for a waiter before eviction, and how + /// long an orphan stays with no live waiter. `None` takes + /// [`super::completion_gc::DEFAULT_WAITERLESS_TTL`]. pub(crate) waiterless_ttl: Option, } impl Inner { - /// Remove `txn`'s entry and deliver `outcome` to `waiter`. `signal` names - /// the event in the warning logged when the receiver is gone. - pub(crate) fn fire( - &mut self, - txn: TxnId, - waiter: CompletionWaiter, - outcome: AttemptOutcome, - ack_results: Vec>, - signal: &'static str, - ) { - self.completions.remove(&txn); - if !waiter.send(outcome, ack_results) { - tracing::warn!( - epoch = txn.epoch, - position = txn.position, - signal, - "calvin completion receiver dropped before its outcome fired; \ - client likely timed out on completion wait" - ); + /// Deliver `txn`'s outcome to its waiter once the entry is terminal, then + /// queue or evict waiterless terminal entries. + /// + /// The entry stays after its outcome fires. Its votes, verdict, and acks + /// answer the participants on this node that still probe it. The + /// waiterless sweep removes it. + pub(crate) fn settle(&mut self, txn: TxnId) { + if let Some(entry) = self.completions.get_mut(&txn) + && entry.is_terminal() + && let Some(verdict) = entry.verdict + && let Some(waiter) = entry.completion_tx.take() + { + let results = entry.take_ack_results(); + if !waiter.send(outcome_for(verdict), results) { + tracing::warn!( + epoch = txn.epoch, + position = txn.position, + "calvin completion receiver dropped before its outcome fired; \ + client likely timed out on completion wait" + ); + } } + self.settle_waiterless(txn); + } +} + +/// The coordinator's outcome for a stored verdict. +fn outcome_for(verdict: VerdictOutcome) -> AttemptOutcome { + match verdict { + VerdictOutcome::Commit => AttemptOutcome::Completed, + VerdictOutcome::Abort(reason) => AttemptOutcome::Aborted { reason }, } } @@ -223,10 +224,8 @@ impl CalvinCompletionRegistry { client likely timed out on submit_with_retry" ); } - inner - .completions - .entry(txn) - .or_insert_with(|| PendingCompletion::new(expected_participants)); + let entry = inner.entry_mut(txn); + entry.expected_participants = entry.expected_participants.max(expected_participants); } /// Register interest in `txn`'s terminal outcome, seeding the authoritative @@ -260,46 +259,14 @@ impl CalvinCompletionRegistry { rx } + /// Store `tx` as `txn`'s waiter. An entry that is terminal already, from + /// a verdict and acks that raced ahead of registration, fires at once. fn register_waiter(&self, txn: TxnId, expected_participants: usize, tx: CompletionWaiter) { let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); - let entry = inner - .completions - .entry(txn) - .or_insert_with(|| PendingCompletion::new(expected_participants)); + let entry = inner.entry_mut(txn); entry.expected_participants = entry.expected_participants.max(expected_participants); - // Routing failure takes precedence over everything else: it is terminal - // and must never be masked by a later ack or mismatch signal. - if let Some(detail) = entry.routing_failed.take() { - inner.fire( - txn, - tx, - AttemptOutcome::Failed { detail }, - Vec::new(), - "routing failure", - ); - } else if entry.abandoned { - // The entry keeps its stored verdict for the participants that - // still probe it, and parks as a waiterless one. - super::completion_parts::send_parts_lost(txn, tx); - inner.settle_waiterless(txn); - } else if entry.mismatched { - inner.fire( - txn, - tx, - AttemptOutcome::Mismatch, - Vec::new(), - "OLLP mismatch", - ); - } else if entry.is_complete() { - // Acks raced ahead of waiter registration: consult the stored - // verdict so an already-complete ABORT surfaces as `Aborted`, not a - // false `Completed`. - let outcome = outcome_for(entry); - let results = entry.take_ack_results(); - inner.fire(txn, tx, outcome, results, "all acked"); - } else { - entry.completion_tx = Some(tx); - } + entry.completion_tx = Some(tx); + inner.settle(txn); } pub fn note_completion_ack(&self, txn: TxnId, vshard_id: u32) { @@ -310,94 +277,20 @@ impl CalvinCompletionRegistry { /// it carried. An empty `result` records none. pub fn note_completion_ack_with(&self, txn: TxnId, vshard_id: u32, result: Vec) { let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); - let entry = inner - .completions - .entry(txn) - .or_insert_with(|| PendingCompletion::new(0)); + let entry = inner.entry_mut(txn); entry.acked_vshards.insert(vshard_id); if !result.is_empty() { entry.ack_results.insert(vshard_id, result); } - if entry.is_complete() { - // Consult the stored global verdict: `Verdict` is applied strictly - // before this abort's `CompletionAck` in the sequencer Raft log, so - // an ABORT is `Some(false)` here and must surface as `Aborted` - // rather than a silent `Completed` (which would drop the writes and - // report COMMIT SUCCESS to the client). - // - // Only fire + evict when the coordinator's waiter is registered. If - // the final ack races AHEAD of registration, LEAVE the entry — its - // `acked_vshards` (and the stored verdict) persist, so - // `register_completion`'s `is_complete()` branch fires the outcome. - // Evicting here would strand that branch (a fresh entry created by - // the later `register_completion` never re-completes). Mirrors how - // `mismatched`/`routing_failed` persist across the same race. - if let Some(tx) = entry.completion_tx.take() { - let outcome = outcome_for(entry); - let results = entry.take_ack_results(); - inner.fire(txn, tx, outcome, results, "final ack"); - } - } - inner.settle_waiterless(txn); - } - - /// Record an OLLP prediction mismatch for `txn`, the second terminal outcome - /// of an attempt. Mismatch takes precedence over completion: a mismatched - /// attempt must retry, never falsely report success. - /// - /// If the coordinator's waiter is already registered, fire `Mismatch` and - /// evict the entry. Otherwise leave the `mismatched` flag set so a later - /// `register_completion` fires it (mirrors `acked_vshards` persistence). - pub fn note_ollp_mismatch(&self, txn: TxnId) { - let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); - let entry = inner - .completions - .entry(txn) - .or_insert_with(|| PendingCompletion::new(0)); - entry.mismatched = true; - if let Some(tx) = entry.completion_tx.take() { - inner.fire( - txn, - tx, - AttemptOutcome::Mismatch, - Vec::new(), - "OLLP mismatch", - ); - } - inner.settle_waiterless(txn); - } - - /// Record a terminal, NON-retryable routing failure for `txn` — the - /// scheduler rejected the transaction's local plan routing as - /// `Unroutable`, `ControlPlaneOnly`, or `NotAWrite`. Takes precedence over - /// completion AND over an OLLP mismatch: a routing failure can never - /// converge via retry, so it must never be masked by a later ack or - /// mismatch signal. - /// - /// If the coordinator's waiter is already registered, fire `Failed` and - /// evict the entry. Otherwise leave the `routing_failed` detail set so a - /// later `register_completion` fires it (mirrors `mismatched` persistence). - pub fn note_routing_failed(&self, txn: TxnId, detail: String) { - let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); - let entry = inner - .completions - .entry(txn) - .or_insert_with(|| PendingCompletion::new(0)); - entry.routing_failed = Some(detail.clone()); - if let Some(tx) = entry.completion_tx.take() { - inner.fire( - txn, - tx, - AttemptOutcome::Failed { detail }, - Vec::new(), - "routing failure", - ); - } - inner.settle_waiterless(txn); + // A participant acks only after it read the verdict, so the verdict + // is stored before the last ack applies. A waiter that registers + // after the last ack finds the terminal entry and fires then. + inner.settle(txn); } /// Set how long a terminal entry with no waiter stays for one to - /// register. The host sets it longer than its statement deadline. + /// register, and how long an orphan stays with no live waiter. The host + /// sets it longer than its statement deadline. pub fn set_waiterless_ttl(&self, ttl: Duration) { self.inner .lock() @@ -408,7 +301,7 @@ impl CalvinCompletionRegistry { /// What this registry holds for participant `vshard` of `txn`. /// /// `None` means no entry exists for `txn`. That is either a txn this node - /// never seeded, or one whose outcome already fired and evicted its entry. + /// never seeded, or a terminal one the waiterless sweep evicted. pub fn participant_progress(&self, txn: TxnId, vshard: u32) -> Option { self.inner .lock() @@ -419,13 +312,10 @@ impl CalvinCompletionRegistry { voted: entry.votes.contains_key(&vshard), acked: entry.acked_vshards.contains(&vshard), has_verdict: entry.verdict.is_some(), - mismatched: entry.mismatched, - routing_failed: entry.routing_failed.is_some(), }) } - /// Test-only: returns the number of pending completion entries. - /// Used to verify entries are removed once all acks arrive (no leak). + /// Test-only: returns the number of completion entries held. #[cfg(test)] pub fn pending_completions_len(&self) -> usize { self.inner @@ -450,31 +340,37 @@ impl CalvinCompletionRegistry { mod tests { use super::*; - /// A replica on which no coordinator waits keeps no completion entry, - /// and no ack result, past the eviction window. An entry that has a - /// waiter stays until its outcome fires. - #[tokio::test] - async fn waiterless_terminal_entries_are_evicted_with_their_results() { + /// A registry that evicts a waiterless terminal entry at its next change. + fn evicting_registry() -> Arc { let reg = CalvinCompletionRegistry::new_detached(); reg.set_waiterless_ttl(Duration::ZERO); + reg + } + + /// A replica on which no coordinator waits keeps no terminal entry, and + /// no ack result, past the eviction window. An entry that has a waiter, + /// or that is sequenced and not terminal, stays. + #[tokio::test] + async fn waiterless_sweep_evicts_only_terminal_entries() { + let reg = evicting_registry(); let acked = TxnId::new(3, 0); - reg.note_assigned(1, acked, 1); + reg.seed_expected(acked, 1); + reg.note_verdict(acked, VerdictOutcome::Commit); reg.note_completion_ack_with(acked, 5, vec![0xAB; 4096]); assert_eq!( reg.pending_completions_len(), 0, - "a complete entry nobody waits for is evicted with its result" + "a terminal entry nobody waits for is evicted with its result" ); - let mismatched = TxnId::new(3, 1); - reg.note_ollp_mismatch(mismatched); - assert_eq!(reg.pending_completions_len(), 0); - let awaited = TxnId::new(3, 2); + let undecided = TxnId::new(3, 3); let rx = reg.register_completion_report(awaited, 2); + reg.note_verdict(awaited, VerdictOutcome::Commit); reg.note_completion_ack_with(awaited, 5, b"five".to_vec()); - reg.note_completion_ack_with(TxnId::new(3, 3), 6, b"other".to_vec()); + reg.seed_expected(undecided, 2); + reg.note_completion_ack_with(undecided, 6, b"other".to_vec()); assert!( reg.participant_progress(awaited, 5).is_some(), "an entry with a waiter is never evicted" @@ -482,6 +378,12 @@ mod tests { reg.note_completion_ack_with(awaited, 6, b"six".to_vec()); let report = rx.await.expect("completion fires"); assert_eq!(report.ack_results, vec![b"five".to_vec(), b"six".to_vec()]); + assert_eq!( + reg.pending_completions_len(), + 1, + "only the undecided entry stays" + ); + assert!(reg.participant_progress(undecided, 6).is_some()); } /// The coordinator hosts no replica of either participant: it learns @@ -490,8 +392,10 @@ mod tests { /// vShard order, whether the acks land before or after it registers. #[tokio::test] async fn a_report_carries_every_participants_ack_result() { - let reg = CalvinCompletionRegistry::new_detached(); + let reg = evicting_registry(); let txn = TxnId::new(11, 4); + reg.seed_expected(txn, 3); + reg.note_verdict(txn, VerdictOutcome::Commit); reg.note_completion_ack_with(txn, 20, b"twenty".to_vec()); let rx = reg.register_completion_report(txn, 3); reg.note_completion_ack_with(txn, 10, b"ten".to_vec()); @@ -504,89 +408,80 @@ mod tests { ); assert_eq!(reg.pending_completions_len(), 0); + // A terminal entry waits the window for its waiter. + let reg = CalvinCompletionRegistry::new_detached(); let late = TxnId::new(11, 5); + reg.seed_expected(late, 1); + reg.note_verdict(late, VerdictOutcome::Commit); reg.note_completion_ack_with(late, 7, b"seven".to_vec()); let report = reg .register_completion_report(late, 1) .await - .expect("an already complete txn fires on registration"); + .expect("an already terminal txn fires on registration"); assert_eq!(report.ack_results, vec![b"seven".to_vec()]); } #[tokio::test] - async fn completion_entry_removed_after_all_acks() { - let reg = CalvinCompletionRegistry::new_detached(); + async fn completion_fires_once_the_verdict_and_every_ack_are_in() { + let reg = evicting_registry(); let txn = TxnId::new(7, 0); reg.note_assigned(1, txn, 2); - let rx = reg.register_completion(txn, 2); - assert_eq!(reg.pending_completions_len(), 1); + let mut rx = reg.register_completion(txn, 2); + reg.note_verdict(txn, VerdictOutcome::Commit); reg.note_completion_ack(txn, 10); - assert_eq!(reg.pending_completions_len(), 1); + assert_eq!(rx.try_recv(), Err(oneshot::error::TryRecvError::Empty)); reg.note_completion_ack(txn, 20); - let outcome = rx.await.expect("completion fires"); - assert_eq!(outcome, AttemptOutcome::Completed); assert_eq!( - reg.pending_completions_len(), - 0, - "entry must be evicted once all expected vshards have acked" + rx.await.expect("completion fires"), + AttemptOutcome::Completed ); + assert_eq!(reg.pending_completions_len(), 0); } #[tokio::test] - async fn completion_entry_removed_when_register_arrives_after_acks() { - let reg = CalvinCompletionRegistry::new_detached(); + async fn completion_fires_when_register_arrives_between_acks() { + let reg = evicting_registry(); let txn = TxnId::new(9, 3); - reg.note_assigned(1, txn, 2); + reg.seed_expected(txn, 2); + reg.note_verdict(txn, VerdictOutcome::Commit); reg.note_completion_ack(txn, 10); - // Entry remains: expected=2, only 1 ack received. assert_eq!(reg.pending_completions_len(), 1); let rx = reg.register_completion(txn, 2); - assert_eq!(reg.pending_completions_len(), 1); reg.note_completion_ack(txn, 20); let outcome = rx.await.expect("completion fires once both acks arrived"); assert_eq!(outcome, AttemptOutcome::Completed); - assert_eq!( - reg.pending_completions_len(), - 0, - "entry must be evicted once awaiter is signalled" - ); + assert_eq!(reg.pending_completions_len(), 0); } + /// Every ack without a stored verdict is not terminal: the waiter keeps + /// waiting, and the sweep keeps the entry. #[tokio::test] - async fn mismatch_arriving_before_register_fires_mismatch() { - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(11, 1); + async fn all_acks_without_a_verdict_are_not_terminal() { + let reg = evicting_registry(); + let txn = TxnId::new(34, 3); reg.note_assigned(1, txn, 2); - // Mismatch observed before the coordinator registers its waiter: the - // flag must persist so a later register_completion fires it. - reg.note_ollp_mismatch(txn); + let mut rx = reg.register_completion(txn, 2); + reg.note_completion_ack(txn, 10); + reg.note_completion_ack(txn, 20); + assert_eq!(rx.try_recv(), Err(oneshot::error::TryRecvError::Empty)); assert_eq!(reg.pending_completions_len(), 1); - let rx = reg.register_completion(txn, 2); - let outcome = rx.await.expect("mismatch fires"); - assert_eq!(outcome, AttemptOutcome::Mismatch); + + reg.note_verdict(txn, VerdictOutcome::Commit); assert_eq!( - reg.pending_completions_len(), - 0, - "entry must be evicted once mismatch is signalled" + rx.await.expect("completion fires"), + AttemptOutcome::Completed ); } + /// A waiterless entry with every ack but no verdict survives the sweep. #[tokio::test] - async fn register_before_mismatch_fires_mismatch() { - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(12, 5); - reg.note_assigned(1, txn, 2); - let rx = reg.register_completion(txn, 2); - assert_eq!(reg.pending_completions_len(), 1); - // Waiter already stored; the mismatch must wake it directly. - reg.note_ollp_mismatch(txn); - let outcome = rx.await.expect("mismatch fires"); - assert_eq!(outcome, AttemptOutcome::Mismatch); - assert_eq!( - reg.pending_completions_len(), - 0, - "entry must be evicted once mismatch is signalled" - ); + async fn a_waiterless_entry_without_a_verdict_is_never_evicted() { + let reg = evicting_registry(); + let txn = TxnId::new(34, 4); + reg.seed_expected(txn, 1); + reg.note_completion_ack(txn, 10); + reg.note_completion_ack(TxnId::new(34, 5), 10); + assert!(reg.participant_progress(txn, 10).is_some()); } #[tokio::test] @@ -595,9 +490,10 @@ mod tests { // register_completion must seed expected_participants from the assignment. // Without the seed (or with the is_complete>0 guard absent) this would // spuriously fire Completed with zero acks. - let reg = CalvinCompletionRegistry::new_detached(); + let reg = evicting_registry(); let txn = TxnId::new(21, 0); let rx = reg.register_completion(txn, 1); + reg.note_verdict(txn, VerdictOutcome::Commit); assert_eq!( reg.pending_completions_len(), 1, @@ -613,10 +509,11 @@ mod tests { async fn ack_racing_ahead_of_register_does_not_prematurely_complete() { // The replicated ack can reach a remote coordinator's registry BEFORE the // coordinator calls register_completion. With expected_participants still - // unknown (0), the ack must persist without firing/evicting; the later - // register_completion seeds the count and then completes. + // unknown (0), the ack must persist without firing or eviction within the + // window; the later register_completion seeds the count and then completes. let reg = CalvinCompletionRegistry::new_detached(); let txn = TxnId::new(22, 0); + reg.note_verdict(txn, VerdictOutcome::Commit); reg.note_completion_ack(txn, 7); assert_eq!( reg.pending_completions_len(), @@ -626,141 +523,149 @@ mod tests { let rx = reg.register_completion(txn, 1); let outcome = rx.await.expect("completion fires once participants seeded"); assert_eq!(outcome, AttemptOutcome::Completed); + reg.set_waiterless_ttl(Duration::ZERO); + reg.note_completion_ack(TxnId::new(22, 1), 7); assert_eq!(reg.pending_completions_len(), 0); } #[tokio::test] - async fn routing_failed_arriving_before_register_fires_failed() { - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(14, 1); - reg.note_assigned(1, txn, 2); - // Routing failure observed before the coordinator registers its - // waiter: the detail must persist so a later register_completion - // fires it. - reg.note_routing_failed(txn, "unroutable plan".to_owned()); - assert_eq!(reg.pending_completions_len(), 1); + async fn abort_verdict_makes_completion_report_aborted() { + // An ABORT verdict is stored (Raft-ordered) before the acks that complete + // the entry. Completion must report `Aborted`, never a silent + // `Completed` that would drop the writes and report COMMIT success. + let reg = evicting_registry(); + let txn = TxnId::new(31, 0); + reg.seed_expected(txn, 2); + reg.note_verdict( + txn, + VerdictOutcome::Abort(AbortReason::SerializationConflict), + ); let rx = reg.register_completion(txn, 2); - let outcome = rx.await.expect("routing failure fires"); + reg.note_completion_ack(txn, 10); + reg.note_completion_ack(txn, 20); + let outcome = rx.await.expect("completion fires"); assert_eq!( outcome, - AttemptOutcome::Failed { - detail: "unroutable plan".to_owned() - } - ); - assert_eq!( - reg.pending_completions_len(), - 0, - "entry must be evicted once routing failure is signalled" + AttemptOutcome::Aborted { + reason: AbortReason::SerializationConflict + }, + "a stored ABORT verdict must surface as Aborted, not Completed" ); + assert_eq!(reg.pending_completions_len(), 0); } + /// The verdict and acks race ahead of waiter registration. The terminal + /// entry waits the window for its waiter, and registration reports + /// `Aborted` at once. The sweep evicts the entry after its window. #[tokio::test] - async fn register_before_routing_failed_fires_failed() { + async fn abort_verdict_reports_aborted_when_register_arrives_after_acks() { let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(15, 4); - reg.note_assigned(1, txn, 2); - let rx = reg.register_completion(txn, 2); - assert_eq!(reg.pending_completions_len(), 1); - // Waiter already stored; the routing failure must wake it directly. - reg.note_routing_failed(txn, "control-plane-only plan".to_owned()); - let outcome = rx.await.expect("routing failure fires"); + let txn = TxnId::new(32, 1); + reg.seed_expected(txn, 2); + reg.note_verdict(txn, VerdictOutcome::Abort(AbortReason::PlanRejected)); + reg.note_completion_ack(txn, 10); + reg.note_completion_ack(txn, 20); + let mut rx = reg.register_completion(txn, 2); assert_eq!( - outcome, - AttemptOutcome::Failed { - detail: "control-plane-only plan".to_owned() - } + rx.try_recv(), + Ok(AttemptOutcome::Aborted { + reason: AbortReason::PlanRejected + }), + "registration on a terminal entry fires at once" ); assert_eq!( reg.pending_completions_len(), - 0, - "entry must be evicted once routing failure is signalled" + 1, + "the entry waits its window" ); + + reg.set_waiterless_ttl(Duration::ZERO); + reg.note_completion_ack(TxnId::new(32, 2), 10); + assert_eq!(reg.pending_completions_len(), 0); } + /// An ack or verdict that applies after the sweep evicted its terminal + /// entry creates an orphan. The sweep evicts the orphan once its window + /// passed, so late signals never pile up. #[tokio::test] - async fn routing_failed_takes_precedence_over_pending_acks_and_mismatch() { - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(16, 0); - reg.note_assigned(1, txn, 2); - // No waiter registered yet: an ack and a mismatch both persist onto - // the entry without firing anything. + async fn a_late_signal_for_an_evicted_txn_leaves_no_entry() { + let reg = evicting_registry(); + let txn = TxnId::new(35, 0); + reg.seed_expected(txn, 1); + reg.note_vote(txn, 10, ParticipantVote::Commit); + reg.note_verdict(txn, VerdictOutcome::Commit); reg.note_completion_ack(txn, 10); - reg.note_ollp_mismatch(txn); - assert_eq!(reg.pending_completions_len(), 1); - // The routing failure also persists (still no waiter)... - reg.note_routing_failed(txn, "non-write plan".to_owned()); - // ...and when the coordinator finally registers, it must observe the - // routing failure, not the mismatch or the ack — routing failure is - // terminal and must never be masked by either. - let rx = reg.register_completion(txn, 2); - let outcome = rx.await.expect("routing failure fires"); assert_eq!( - outcome, - AttemptOutcome::Failed { - detail: "non-write plan".to_owned() - } + reg.pending_completions_len(), + 0, + "the terminal entry is evicted" + ); + + reg.note_completion_ack(txn, 10); + assert_eq!( + reg.pending_completions_len(), + 0, + "a late ack leaves no entry" ); + reg.note_vote(txn, 10, ParticipantVote::Commit); + reg.note_verdict(txn, VerdictOutcome::Commit); assert_eq!( reg.pending_completions_len(), 0, - "entry must be evicted once routing failure is signalled" + "a late vote and verdict leave no entry" ); + assert_eq!(reg.participant_progress(txn, 10), None); } + /// An orphan a coordinator waits on stays while the coordinator listens, + /// past its window. Once the coordinator drops its receiver, the sweep + /// evicts the orphan. #[tokio::test] - async fn abort_verdict_makes_completion_report_aborted() { - // An ABORT verdict is stored (Raft-ordered) before the acks that complete - // the tally. Completion must consult it and report `Aborted`, never a - // silent `Completed` that would drop the writes and report COMMIT success. - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(31, 0); - reg.note_assigned(1, txn, 2); - reg.note_verdict( - txn, - VerdictOutcome::Abort(AbortReason::SerializationConflict), - ); + async fn an_orphan_stays_while_its_coordinator_listens() { + let reg = evicting_registry(); + let txn = TxnId::new(36, 0); let rx = reg.register_completion(txn, 2); reg.note_completion_ack(txn, 10); - reg.note_completion_ack(txn, 20); - let outcome = rx.await.expect("completion fires"); - assert_eq!( - outcome, - AttemptOutcome::Aborted { - reason: AbortReason::SerializationConflict - }, - "a stored ABORT verdict must surface as Aborted, not Completed" - ); + reg.note_completion_ack(TxnId::new(36, 1), 10); + let progress = reg + .participant_progress(txn, 10) + .expect("a listening coordinator keeps its orphan"); + assert!(progress.acked && !progress.has_verdict); + + drop(rx); + reg.note_completion_ack(TxnId::new(36, 2), 10); + assert_eq!(reg.participant_progress(txn, 10), None); assert_eq!(reg.pending_completions_len(), 0); } + /// A sequenced entry is never an orphan. With no waiter and no verdict, + /// it stays past the window: its participants still need its tally. #[tokio::test] - async fn abort_verdict_reports_aborted_when_register_arrives_after_acks() { - // Acks (and the verdict) race ahead of waiter registration: the - // already-complete branch in `register_completion` must also consult the - // stored verdict and report `Aborted`. - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(32, 1); + async fn a_sequenced_entry_is_never_evicted_as_an_orphan() { + let reg = evicting_registry(); + let txn = TxnId::new(37, 0); reg.note_assigned(1, txn, 2); - reg.note_verdict(txn, VerdictOutcome::Abort(AbortReason::ParticipantError)); - reg.note_completion_ack(txn, 10); - reg.note_completion_ack(txn, 20); - let rx = reg.register_completion(txn, 2); - let outcome = rx.await.expect("completion fires"); + reg.note_completion_ack(TxnId::new(37, 1), 10); assert_eq!( - outcome, - AttemptOutcome::Aborted { - reason: AbortReason::ParticipantError - } + reg.participant_progress(txn, 10), + None, + "an assignment alone does not sequence the entry" + ); + + reg.seed_expected(txn, 2); + reg.note_vote(txn, 10, ParticipantVote::Commit); + reg.note_completion_ack(TxnId::new(37, 2), 10); + assert!( + reg.participant_progress(txn, 10) + .is_some_and(|progress| progress.voted) ); - assert_eq!(reg.pending_completions_len(), 0); } #[tokio::test] async fn commit_verdict_reports_completed() { - // A COMMIT verdict must still report `Completed`. - let reg = CalvinCompletionRegistry::new_detached(); + let reg = evicting_registry(); let txn = TxnId::new(33, 2); - reg.note_assigned(1, txn, 2); + reg.seed_expected(txn, 2); reg.note_verdict(txn, VerdictOutcome::Commit); let rx = reg.register_completion(txn, 2); reg.note_completion_ack(txn, 10); @@ -770,6 +675,37 @@ mod tests { assert_eq!(reg.pending_completions_len(), 0); } + /// A fired outcome keeps the entry's votes, verdict, and acks for the + /// participants that still probe it, until the sweep evicts it. + #[tokio::test] + async fn a_fired_outcome_keeps_the_tally_until_the_sweep() { + let reg = CalvinCompletionRegistry::new_detached(); + let txn = TxnId::new(40, 2); + reg.seed_expected(txn, 1); + let rx = reg.register_completion(txn, 1); + reg.note_vote(txn, 1, ParticipantVote::Abort(AbortReason::PlanRejected)); + reg.note_verdict(txn, VerdictOutcome::Abort(AbortReason::PlanRejected)); + reg.note_completion_ack(txn, 1); + assert_eq!( + rx.await.expect("completion fires"), + AttemptOutcome::Aborted { + reason: AbortReason::PlanRejected + } + ); + + let progress = reg.participant_progress(txn, 1).expect("entry stays"); + assert!(progress.voted && progress.acked && progress.has_verdict); + assert_eq!(reg.verdict(txn), Some(false)); + assert_eq!( + reg.vote_tally(txn).and_then(|tally| tally.get(&1).copied()), + Some(ParticipantVote::Abort(AbortReason::PlanRejected)) + ); + + reg.set_waiterless_ttl(Duration::ZERO); + reg.note_completion_ack(TxnId::new(40, 3), 1); + assert_eq!(reg.participant_progress(txn, 1), None); + } + /// A dropped assignment closes the caller's channel at once and leaves no /// sender behind. A later `note_assigned` for the seq finds none. #[test] @@ -796,41 +732,6 @@ mod tests { assert_eq!(reg.pending_assignments_len(), 0); } - #[tokio::test] - async fn no_verdict_reports_completed_unchanged() { - // Single-shard / no-verdict paths (`verdict == None`) must report - // `Completed` unchanged — the fix must not stall them. - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(34, 3); - reg.note_assigned(1, txn, 2); - let rx = reg.register_completion(txn, 2); - reg.note_completion_ack(txn, 10); - reg.note_completion_ack(txn, 20); - let outcome = rx.await.expect("completion fires"); - assert_eq!(outcome, AttemptOutcome::Completed); - assert_eq!(reg.pending_completions_len(), 0); - } - - #[tokio::test] - async fn mismatch_takes_precedence_over_pending_acks() { - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(13, 2); - reg.note_assigned(1, txn, 2); - let rx = reg.register_completion(txn, 2); - // One ack arrives but the attempt is not yet complete (expected=2). - reg.note_completion_ack(txn, 10); - assert_eq!(reg.pending_completions_len(), 1); - // A mismatch on the same attempt must win and force a retry. - reg.note_ollp_mismatch(txn); - let outcome = rx.await.expect("mismatch fires"); - assert_eq!(outcome, AttemptOutcome::Mismatch); - assert_eq!( - reg.pending_completions_len(), - 0, - "entry must be evicted once mismatch is signalled" - ); - } - #[tokio::test] async fn participant_progress_is_none_before_any_entry_exists() { let reg = CalvinCompletionRegistry::new_detached(); @@ -844,7 +745,6 @@ mod tests { reg.seed_expected(txn, 2); let empty = reg.participant_progress(txn, 1).expect("seeded entry"); assert!(!empty.voted && !empty.acked && !empty.has_verdict); - assert!(!empty.mismatched && !empty.routing_failed); reg.note_vote(txn, 1, ParticipantVote::Commit); reg.note_completion_ack(txn, 1); @@ -857,18 +757,7 @@ mod tests { ); reg.note_verdict(txn, VerdictOutcome::Commit); - reg.note_ollp_mismatch(txn); - reg.note_routing_failed(txn, "unroutable".to_string()); let txn_wide = reg.participant_progress(txn, 2).expect("entry"); - assert!(txn_wide.has_verdict && txn_wide.mismatched && txn_wide.routing_failed); - } - - #[tokio::test] - async fn participant_progress_is_none_once_the_outcome_fired() { - let reg = CalvinCompletionRegistry::new_detached(); - let txn = TxnId::new(40, 2); - let _rx = reg.register_completion(txn, 1); - reg.note_completion_ack(txn, 1); - assert_eq!(reg.participant_progress(txn, 1), None); + assert!(txn_wide.has_verdict); } } diff --git a/nodedb-cluster/src/calvin/completion_entry.rs b/nodedb-cluster/src/calvin/completion_entry.rs index 60f64a2a6..b8b5a3ce0 100644 --- a/nodedb-cluster/src/calvin/completion_entry.rs +++ b/nodedb-cluster/src/calvin/completion_entry.rs @@ -1,11 +1,12 @@ // SPDX-License-Identifier: BUSL-1.1 //! One transaction's completion entry in the -//! [`super::completion::CalvinCompletionRegistry`], and the outcome it fires. +//! [`super::completion::CalvinCompletionRegistry`]. use std::collections::{BTreeMap, BTreeSet}; +use std::time::Instant; -use super::completion::{AttemptOutcome, ParticipantVote, VerdictOutcome}; +use super::completion::{ParticipantVote, VerdictOutcome}; use super::completion_waiter::CompletionWaiter; pub(crate) struct PendingCompletion { @@ -15,15 +16,6 @@ pub(crate) struct PendingCompletion { /// by vShard. A participant whose ack carried none has no entry. pub(crate) ack_results: BTreeMap>, pub(crate) completion_tx: Option, - /// Set when an OLLP mismatch is observed before the coordinator registers - /// its waiter, so the outcome is not lost across registration order (mirrors - /// how `acked_vshards` persists ack state regardless of registration order). - pub(crate) mismatched: bool, - /// Set when a terminal routing failure is observed before the coordinator - /// registers its waiter, mirroring `mismatched`. Takes precedence over both - /// `mismatched` and completion: a routing failure is never retried and never - /// falsely reported as success. - pub(crate) routing_failed: Option, /// Set when the multi-part transaction lost its parts /// (`SequencerEntry::TxnPartsAbandoned`). Its outcome is /// `Aborted { PartsLost }` without waiting for acks: no participant @@ -52,39 +44,29 @@ pub(crate) struct PendingCompletion { /// Queued for eviction as a terminal entry with no waiter /// (`completion_gc`). pub(crate) parked: bool, -} - -/// Choose the terminal outcome for a COMPLETED entry (all expected vshards -/// acked) by consulting the durable global verdict. -/// -/// The verdict is applied from a replicated `SequencerEntry::Verdict` that is -/// strictly ordered BEFORE the abort's `CompletionAck` in the sequencer Raft -/// log, so an ABORT verdict is always stored by the time this entry's -/// completion fires. A stored abort becomes `Aborted`, carrying the reason the -/// verdict recorded; `None` (single-shard / no-verdict paths) and a commit -/// verdict are `Completed`. We deliberately do NOT gate on `verdict.is_some()` -/// — that would stall the no-verdict completion paths. -pub(crate) fn outcome_for(entry: &PendingCompletion) -> AttemptOutcome { - match entry.verdict { - Some(VerdictOutcome::Abort(reason)) => AttemptOutcome::Aborted { reason }, - _ => AttemptOutcome::Completed, - } + /// Set when this replica applied the `EpochBatch` that sequenced the + /// txn (`seed_expected`). An entry without it is an orphan: the + /// waiterless sweep evicts it once it outlives the window with no live + /// waiter (`completion_gc`). + pub(crate) sequenced: bool, + /// When this entry was created. + pub(crate) created: Instant, } impl PendingCompletion { - pub(crate) fn new(expected_participants: usize) -> Self { + pub(crate) fn new(created: Instant) -> Self { Self { - expected_participants, + expected_participants: 0, acked_vshards: BTreeSet::new(), ack_results: BTreeMap::new(), completion_tx: None, - mismatched: false, - routing_failed: None, abandoned: false, votes: BTreeMap::new(), verdict: None, verdict_proposed: false, parked: false, + sequenced: false, + created, } } @@ -92,10 +74,20 @@ impl PendingCompletion { self.completion_tx.is_some() } - /// Whether the entry's outcome is decided: every expected participant - /// acked, or an OLLP mismatch or a routing failure is recorded. + /// Whether a waiter is registered and its receiver still listens. + pub(crate) fn has_live_waiter(&self) -> bool { + self.completion_tx + .as_ref() + .is_some_and(CompletionWaiter::is_live) + } + + /// Whether the entry's outcome is decided: the verdict is stored and + /// every expected participant acked, or the transaction lost its parts. + /// + /// Acks without a verdict are never terminal. Every participant acks + /// only after it read the verdict. pub(crate) fn is_terminal(&self) -> bool { - self.is_complete() || self.mismatched || self.routing_failed.is_some() || self.abandoned + self.abandoned || (self.verdict.is_some() && self.is_complete()) } /// Every acked participant's apply result, in vShard order. diff --git a/nodedb-cluster/src/calvin/completion_gc.rs b/nodedb-cluster/src/calvin/completion_gc.rs index 6d2eb9373..1c8d1f7a2 100644 --- a/nodedb-cluster/src/calvin/completion_gc.rs +++ b/nodedb-cluster/src/calvin/completion_gc.rs @@ -10,21 +10,83 @@ //! within its statement deadline, so a terminal entry that gained no waiter //! within the eviction window never will, and is removed with its results. //! -//! An entry becomes terminal when every expected participant acked, or when -//! an OLLP mismatch or a routing failure is recorded for it. A terminal -//! entry with no waiter joins a queue stamped with the instant it did. Each -//! later registry change sweeps the queue front, so the queue holds at most -//! the entries of one window. +//! An entry becomes terminal when its verdict is stored and every expected +//! participant acked, or when the transaction lost its parts. An entry whose +//! outcome fired stays too, for the participants that still probe its +//! verdict. A terminal entry with no waiter joins a queue stamped with the +//! instant it did. Each later registry change sweeps the queue front, so the +//! queue holds at most the entries of one window. A non-terminal sequenced +//! entry is never evicted: the scheduler that holds its txn reports the +//! stall. +//! +//! # Orphans +//! +//! The `EpochBatch` apply seeds a txn's entry on every replica, and marks it +//! sequenced, before it fans the txn out. Every vote, verdict, and ack of the +//! txn follows its `EpochBatch` in the sequencer log. A signal therefore +//! creates an unsequenced entry only in these cases: +//! +//! - It applies after the waiterless sweep evicted the terminal entry. +//! - Its `EpochBatch` applied before this replica's registry existed, in a +//! snapshot or an earlier process. +//! - A coordinator registered, or the leader assigned, before the +//! `EpochBatch` applied here, or for a batch that never commits. +//! +//! No local participant holds such a txn: a scheduler receives a txn only +//! from this replica's `EpochBatch` apply, or from a replay of an applied +//! entry. Only a coordinator can wait on an orphan. Each orphan joins a +//! second queue at creation. The sweep evicts it once the window passed +//! since its creation and no live waiter listens. A waiter that listens +//! re-queues it. +//! +//! The rule never evicts an entry a live coordinator waits on: +//! +//! - A registered waiter whose receiver still listens re-queues the entry. +//! - A coordinator registers within its statement deadline of its submit. A +//! signal of the txn exists only after the submit, so registration lands +//! inside the window that starts at the orphan's creation. The host sets +//! the window longer than the deadline. +//! +//! A late ack or verdict for an evicted txn creates an orphan with no +//! waiter. The sweep evicts it one window later, so it never accumulates. +use std::collections::btree_map::Entry; use std::time::{Duration, Instant}; use super::completion::{Inner, TxnId}; +use super::completion_entry::PendingCompletion; /// The eviction window when the host sets none. Longer than any statement /// deadline, so a coordinator's waiter always registers inside it. pub const DEFAULT_WAITERLESS_TTL: Duration = Duration::from_secs(120); impl Inner { + /// `txn`'s entry. A missing one is created unsequenced and joins the + /// orphan queue. + pub(crate) fn entry_mut(&mut self, txn: TxnId) -> &mut PendingCompletion { + match self.completions.entry(txn) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(slot) => { + // no-determinism: node-local eviction clock; never in the log. + let now = Instant::now(); + self.orphans.push_back((now, txn)); + slot.insert(PendingCompletion::new(now)) + } + } + } + + /// `txn`'s entry, marked sequenced. A missing one is created without an + /// orphan slot. + pub(crate) fn sequenced_entry_mut(&mut self, txn: TxnId) -> &mut PendingCompletion { + let entry = self + .completions + .entry(txn) + // no-determinism: node-local eviction clock; never in the log. + .or_insert_with(|| PendingCompletion::new(Instant::now())); + entry.sequenced = true; + entry + } + /// Queue `txn`'s entry when it just became a waiterless terminal entry, /// then evict every queued entry whose window passed. pub(crate) fn settle_waiterless(&mut self, txn: TxnId) { @@ -32,6 +94,44 @@ impl Inner { let now = Instant::now(); self.park_waiterless(txn, now); self.sweep_waiterless(now); + self.sweep_orphans(now); + } + + /// The eviction window. + fn window(&self) -> Duration { + self.waiterless_ttl.unwrap_or(DEFAULT_WAITERLESS_TTL) + } + + /// Evict every queued orphan that outlived the window with no live + /// waiter. An orphan whose waiter still listens is queued again. + /// + /// A queue slot is skipped when its entry is sequenced, parked as + /// terminal, or younger than the window. A younger entry is a later + /// incarnation of the txn, and it holds its own slot. + pub(crate) fn sweep_orphans(&mut self, now: Instant) { + let window = self.window(); + let mut waited_on = Vec::new(); + while let Some(&(since, txn)) = self.orphans.front() { + if now.saturating_duration_since(since) < window { + break; + } + self.orphans.pop_front(); + let Some(entry) = self.completions.get(&txn) else { + continue; + }; + if entry.sequenced + || entry.parked + || now.saturating_duration_since(entry.created) < window + { + continue; + } + if entry.has_live_waiter() { + waited_on.push((now, txn)); + continue; + } + self.completions.remove(&txn); + } + self.orphans.extend(waited_on); } /// Queue `txn`'s entry for eviction when it is terminal, has no waiter, @@ -50,7 +150,7 @@ impl Inner { /// Remove every queued entry that stayed waiterless for the whole /// window. pub(crate) fn sweep_waiterless(&mut self, now: Instant) { - let ttl = self.waiterless_ttl.unwrap_or(DEFAULT_WAITERLESS_TTL); + let ttl = self.window(); while let Some(&(since, txn)) = self.waiterless.front() { if now.saturating_duration_since(since) < ttl { break; diff --git a/nodedb-cluster/src/calvin/completion_parts.rs b/nodedb-cluster/src/calvin/completion_parts.rs index c08b128c1..06ba9390e 100644 --- a/nodedb-cluster/src/calvin/completion_parts.rs +++ b/nodedb-cluster/src/calvin/completion_parts.rs @@ -3,8 +3,7 @@ //! The completion registry's side of a multi-part transaction that lost its //! parts. -use super::completion::{AttemptOutcome, CalvinCompletionRegistry, TxnId, VerdictOutcome}; -use super::completion_entry::PendingCompletion; +use super::completion::{CalvinCompletionRegistry, TxnId, VerdictOutcome}; use super::completion_verdict::VerdictSignal; use super::sequencer::AbortReason; @@ -19,20 +18,16 @@ impl CalvinCompletionRegistry { /// waits for no ack: no participant staged the whole transaction, and /// no verdict can ever commit it. /// - /// Idempotent. A later ack of `txn` finds no entry for its outcome and - /// parks as a waiterless one. + /// Idempotent. A later ack of `txn` lands on the terminal entry, which + /// the waiterless sweep removes. pub fn note_parts_abandoned(&self, txn: TxnId) { let verdict = VerdictOutcome::Abort(AbortReason::PartsLost); let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); - let waiter = { - let entry = inner - .completions - .entry(txn) - .or_insert_with(|| PendingCompletion::new(0)); + { + let entry = inner.entry_mut(txn); entry.abandoned = true; entry.verdict = Some(verdict); - entry.completion_tx.take() - }; + } let signal = VerdictSignal { epoch: txn.epoch, position: txn.position, @@ -43,31 +38,14 @@ impl CalvinCompletionRegistry { } // The entry stays: a participant that staged every part targeting // it probes the stored verdict when it parks, after this. - if let Some(waiter) = waiter { - send_parts_lost(txn, waiter); - } - inner.settle_waiterless(txn); - } -} - -/// Answer `waiter` with `Aborted { PartsLost }`, keeping `txn`'s entry and -/// its stored verdict for the participants that still probe it. -pub(crate) fn send_parts_lost(txn: TxnId, waiter: super::completion_waiter::CompletionWaiter) { - let outcome = AttemptOutcome::Aborted { - reason: AbortReason::PartsLost, - }; - if !waiter.send(outcome, Vec::new()) { - tracing::warn!( - epoch = txn.epoch, - position = txn.position, - "calvin completion receiver dropped before its parts-lost outcome fired" - ); + inner.settle(txn); } } #[cfg(test)] mod tests { use super::*; + use crate::calvin::AttemptOutcome; #[tokio::test] async fn lost_parts_abort_a_registered_waiter_without_acks() { diff --git a/nodedb-cluster/src/calvin/completion_verdict.rs b/nodedb-cluster/src/calvin/completion_verdict.rs index 1cf8845b2..8edb5ad14 100644 --- a/nodedb-cluster/src/calvin/completion_verdict.rs +++ b/nodedb-cluster/src/calvin/completion_verdict.rs @@ -12,7 +12,6 @@ use tokio::sync::mpsc; use super::TxnId; use super::completion::{CalvinCompletionRegistry, ParticipantVote, VerdictOutcome}; -use super::completion_entry::PendingCompletion; use super::sequencer::AbortReason; /// Push notification that a staged cross-shard txn's authoritative global @@ -72,10 +71,7 @@ impl CalvinCompletionRegistry { /// no-op (max with the existing value). pub fn seed_expected(&self, txn: TxnId, expected: usize) { let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); - let entry = inner - .completions - .entry(txn) - .or_insert_with(|| PendingCompletion::new(0)); + let entry = inner.sequenced_entry_mut(txn); entry.expected_participants = entry.expected_participants.max(expected); } @@ -88,10 +84,7 @@ impl CalvinCompletionRegistry { /// each parked participant's flush (commit) or drop (abort). pub fn note_vote(&self, txn_id: TxnId, vshard: u32, vote: ParticipantVote) { let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); - let entry = inner - .completions - .entry(txn_id) - .or_insert_with(|| PendingCompletion::new(0)); + let entry = inner.entry_mut(txn_id); entry.votes.insert(vshard, vote); // On the transition to a complete tally, emit the aggregated verdict @@ -164,10 +157,7 @@ impl CalvinCompletionRegistry { pub fn note_verdict(&self, txn: TxnId, verdict: VerdictOutcome) { let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); { - let entry = inner - .completions - .entry(txn) - .or_insert_with(|| PendingCompletion::new(0)); + let entry = inner.entry_mut(txn); match entry.verdict { Some(existing) if existing.is_commit() != verdict.is_commit() => { tracing::warn!( @@ -196,6 +186,8 @@ impl CalvinCompletionRegistry { for tx in inner.verdict_signal_senders.values() { let _ = tx.try_send(signal); } + // A waiter fires here when every ack is in already. + inner.settle(txn); } /// The parked scheduler's flush/drop gate: `Some(true)` commit, `Some(false)` @@ -228,16 +220,18 @@ impl CalvinCompletionRegistry { /// commit, otherwise abort with the highest-precedence reason. /// /// Precedence: `PartsLost` outranks everything, since no participant holds -/// the whole transaction. `ParticipantError` outranks `CollectionSuperseded`, which -/// outranks `PredictionDrift`, which outranks `SerializationConflict`. A peer -/// that never staged makes every other verdict unverifiable, a superseded -/// collection makes a retry against the same read-set pointless, and a -/// drifted prediction retries with a fresh reconnaissance. The tally is +/// the whole transaction. `PlanRejected` outranks `ParticipantError`, which +/// outranks `CollectionSuperseded`, which outranks `PredictionDrift`, which +/// outranks `SerializationConflict`. Rejected plans fail the same way on every +/// resubmit, a peer that never staged makes every other verdict unverifiable, +/// a superseded collection makes a retry against the same read-set pointless, +/// and a drifted prediction retries with a fresh reconnaissance. The tally is /// order-independent. fn tally_verdict(votes: &BTreeMap) -> VerdictOutcome { fn rank(reason: AbortReason) -> u8 { match reason { - AbortReason::PartsLost => 4, + AbortReason::PartsLost => 5, + AbortReason::PlanRejected => 4, AbortReason::ParticipantError => 3, AbortReason::CollectionSuperseded => 2, AbortReason::PredictionDrift => 1, @@ -408,6 +402,36 @@ mod tests { } } + /// A rejected plan outranks every staged abort cause, in any vote order. + #[test] + fn plan_rejected_outranks_staged_abort_causes() { + let rejected = ParticipantVote::Abort(AbortReason::PlanRejected); + let tally = |votes: [&ParticipantVote; 2]| { + let votes: BTreeMap = + (1..).zip(votes.into_iter().copied()).collect(); + tally_verdict(&votes) + }; + for other in [ + AbortReason::SerializationConflict, + AbortReason::PredictionDrift, + AbortReason::CollectionSuperseded, + AbortReason::ParticipantError, + ] { + let other = ParticipantVote::Abort(other); + for votes in [[&rejected, &other], [&other, &rejected]] { + assert_eq!( + tally(votes), + VerdictOutcome::Abort(AbortReason::PlanRejected), + "{other:?}" + ); + } + } + let lost = ParticipantVote::Abort(AbortReason::PartsLost); + for votes in [[&rejected, &lost], [&lost, &rejected]] { + assert_eq!(tally(votes), VerdictOutcome::Abort(AbortReason::PartsLost)); + } + } + #[tokio::test] async fn seed_expected_is_idempotent_max_and_enables_verdict_without_note_assigned() { // Mirrors complete_all_true_tally_emits_commit_verdict_once, but seeds diff --git a/nodedb-cluster/src/calvin/completion_waiter.rs b/nodedb-cluster/src/calvin/completion_waiter.rs index 7faf0edda..31d72861a 100644 --- a/nodedb-cluster/src/calvin/completion_waiter.rs +++ b/nodedb-cluster/src/calvin/completion_waiter.rs @@ -32,6 +32,15 @@ pub(crate) enum CompletionWaiter { } impl CompletionWaiter { + /// Whether the receiver still listens. A coordinator that gave up + /// waiting dropped it. + pub(crate) fn is_live(&self) -> bool { + match self { + Self::Outcome(tx) => !tx.is_closed(), + Self::Report(tx) => !tx.is_closed(), + } + } + /// Deliver `outcome` with `ack_results`. `false` when the receiver is /// gone. pub(crate) fn send(self, outcome: AttemptOutcome, ack_results: Vec>) -> bool { diff --git a/nodedb-cluster/src/calvin/mod.rs b/nodedb-cluster/src/calvin/mod.rs index 7f72bd68d..f46bffc0a 100644 --- a/nodedb-cluster/src/calvin/mod.rs +++ b/nodedb-cluster/src/calvin/mod.rs @@ -18,8 +18,8 @@ pub use completion::{ pub use completion_verdict::VerdictSignal; pub use completion_waiter::CompletionReport; pub use sequencer::{ - AbortReason, AdmittedTx, ConflictKey, CutInstantHook, EpochCheck, Inbox, InboxReceiver, - PartsIntake, PartsOffer, PartsOfferStatus, RejectedTx, ReservationInbox, + AbortReason, AdmittedTx, ConflictKey, CutInstantHook, EpochCheck, HistoryOrigin, Inbox, + InboxReceiver, PartsIntake, PartsOffer, PartsOfferStatus, RejectedTx, ReservationInbox, ReservationInboxReceiver, ReservationRequest, RestorePointHook, SEQUENCER_GROUP_ID, SequencerConfig, SequencerEntry, SequencerError, SequencerHalt, SequencerMetrics, SequencerReceivers, SequencerRestorePoint, SequencerService, SequencerSnapshot, diff --git a/nodedb-cluster/src/calvin/sequencer/entry.rs b/nodedb-cluster/src/calvin/sequencer/entry.rs index 63f5dd4c4..a4e59274a 100644 --- a/nodedb-cluster/src/calvin/sequencer/entry.rs +++ b/nodedb-cluster/src/calvin/sequencer/entry.rs @@ -44,6 +44,10 @@ pub enum AbortReason { /// held them changed before it proposed them all. No participant staged /// the whole transaction, and the coordinator resubmits it. PartsLost, + /// A participant could not decode or route its slice of the plans, or + /// found no local work in them. Every replica rejects the same plans + /// alike, and a resubmit of them fails the same way. + PlanRejected, } /// An entry in the replicated sequencer log. @@ -83,29 +87,8 @@ pub enum SequencerEntry { /// first ack of a vShard in log order answers for it. from_node: u64, }, - /// OLLP predicate-mismatch signal. Proposed by the per-vShard scheduler when - /// the active executor returns `OllpRetryRequired`. Applied on ALL sequencer-group - /// replicas so every node's `CalvinCompletionRegistry` fires `note_ollp_mismatch`, - /// waking the coordinator's retry-loop waiter wherever it is (including remote - /// nodes). Mirrors `CompletionAck` — no `vshard_id` because mismatch is keyed - /// only by `(epoch, position)` like the registry's `TxnId`. - OllpMismatch { epoch: u64, position: u32 }, - /// Local-routing failure signal. Proposed by the per-vShard scheduler when - /// `local_calvin_plans` rejects a plan as `Unroutable`, `ControlPlaneOnly`, - /// or `NotAWrite` — a scheduler-side bug or malformed plan, never a normal - /// retry condition. Applied on ALL sequencer-group replicas so every node's - /// `CalvinCompletionRegistry` fires `note_routing_failed`, waking the - /// coordinator's completion waiter wherever it is with the terminal, - /// NON-retryable `AttemptOutcome::Failed` outcome. Mirrors `OllpMismatch` — - /// no `vshard_id` because the failure is keyed only by `(epoch, position)`. - TxnRoutingFailed { - epoch: u64, - position: u32, - detail: String, - }, /// One participant vShard's durable COMMIT vote for a staged cross-shard - /// txn: its read-set is valid. An abort vote is `AbortVote`. Unlike - /// `OllpMismatch`/`CompletionAck` which key only on `(epoch, position)`, `Vote` + /// txn: its read-set is valid. An abort vote is `AbortVote`. `Vote` /// carries `vshard` because the verdict aggregator must attribute exactly one /// vote per participant to know when the tally is complete. Vote { @@ -282,28 +265,6 @@ mod tests { assert_eq!(batch.epoch, 7); } - #[test] - fn txn_routing_failed_msgpack_roundtrip() { - let entry = SequencerEntry::TxnRoutingFailed { - epoch: 42, - position: 3, - detail: "unroutable plan: Vector".to_owned(), - }; - let bytes = zerompk::to_msgpack_vec(&entry).expect("encode"); - let decoded: SequencerEntry = zerompk::from_msgpack(&bytes).expect("decode"); - let SequencerEntry::TxnRoutingFailed { - epoch, - position, - detail, - } = decoded - else { - panic!("decoded wrong sequencer entry variant"); - }; - assert_eq!(epoch, 42); - assert_eq!(position, 3); - assert_eq!(detail, "unroutable plan: Vector"); - } - #[test] fn vote_msgpack_roundtrip() { let entry = SequencerEntry::Vote { diff --git a/nodedb-cluster/src/calvin/sequencer/mod.rs b/nodedb-cluster/src/calvin/sequencer/mod.rs index 60367d229..c3b9e5c4b 100644 --- a/nodedb-cluster/src/calvin/sequencer/mod.rs +++ b/nodedb-cluster/src/calvin/sequencer/mod.rs @@ -26,7 +26,7 @@ pub use reservation_inbox::{ }; pub use service::{SequencerReceivers, SequencerService}; pub use state_machine::{ - CutInstantHook, RestorePointHook, SequencerRestorePoint, SequencerSnapshot, + CutInstantHook, HistoryOrigin, RestorePointHook, SequencerRestorePoint, SequencerSnapshot, SequencerStateMachine, }; pub use validator::validate_batch; diff --git a/nodedb-cluster/src/calvin/sequencer/replay.rs b/nodedb-cluster/src/calvin/sequencer/replay.rs index 860e67be4..d5cae84de 100644 --- a/nodedb-cluster/src/calvin/sequencer/replay.rs +++ b/nodedb-cluster/src/calvin/sequencer/replay.rs @@ -216,8 +216,6 @@ impl SequencerStateMachine { // scheduler input; they are re-derived via live `apply` on every // replica's completion registry, not replayed to a scheduler. SequencerEntry::CompletionAck { .. } => {} - SequencerEntry::OllpMismatch { .. } => {} - SequencerEntry::TxnRoutingFailed { .. } => {} SequencerEntry::Vote { .. } => {} SequencerEntry::Verdict { .. } => {} SequencerEntry::AbortVote { .. } => {} diff --git a/nodedb-cluster/src/calvin/sequencer/service/core.rs b/nodedb-cluster/src/calvin/sequencer/service/core.rs index f1e7b44d6..b511db8a8 100644 --- a/nodedb-cluster/src/calvin/sequencer/service/core.rs +++ b/nodedb-cluster/src/calvin/sequencer/service/core.rs @@ -79,15 +79,17 @@ pub struct SequencerService { /// The epoch `next_reservation_position` is counting within. When it lags /// the tick's epoch, the band counter is reset before the next mint. pub(super) reservation_epoch: u64, - /// Current epoch number, or `None` until the seed has been derived. + /// The next epoch to mint and the sequencer term it was seeded in, or + /// `None` until the seed has been derived. /// /// The leader starts at the last committed epoch + 1 and increments after /// each successful proposal. The seed is NOT taken at construction — see /// [`Self::ensure_epoch_seeded`] for why it can only be read once the - /// sequencer group's log has been replayed into the state machine. On - /// leader failover, `inbox_receiver` is simply dropped (in-flight + /// sequencer group's log has been replayed into the state machine. A + /// lost leadership clears it, and a cursor from another term is derived + /// again. On leader failover, `inbox_receiver` is dropped (in-flight /// submissions are not in the log and will be retried). - pub(super) current_epoch: Option, + pub(super) current_epoch: Option, /// The `epoch_system_ms` this service last minted, or `None` before its /// first mint. A proposed batch applies later, so the next mint reads /// this as well as the state machine's applied instant. @@ -190,7 +192,7 @@ impl SequencerService { } } => { // Every replica's registry emits deterministically, but only - // the leader proposes — same leader-gate as OllpMismatch/Vote. + // the leader proposes the verdict. if let Some((txn, outcome)) = verdict && self.is_leader() && let Err(e) = self.propose_entry(&verdict_entry(txn, outcome)) @@ -245,6 +247,8 @@ impl SequencerService { // Parts held here are gone with the leadership. The next leader // abandons their transactions. self.drop_part_streams(); + // Another leader can mint epochs before this node leads again. + self.current_epoch = None; debug!( node_id = self.node_id, "not sequencer leader; discarding {discarded} inbox items \ @@ -376,6 +380,7 @@ pub(super) mod tests { use crate::calvin::sequencer::config::SequencerConfig; use crate::calvin::sequencer::inbox::{AdmittedTx, Inbox, new_inbox}; use crate::calvin::sequencer::reservation_inbox::{ReservationInbox, new_reservation_inbox}; + use crate::calvin::sequencer::service::epoch_seed::EpochCursor; use crate::calvin::sequencer::validator::validate_batch; use crate::calvin::types::{ EngineKeySet, EpochBatch, LockKeyWire, ReadWriteSet, ReleaseReason, SequencedTxn, @@ -624,7 +629,7 @@ pub(super) mod tests { } } - fn epoch_batch_bytes(epoch: u64) -> Vec { + pub(in crate::calvin::sequencer::service) fn epoch_batch_bytes(epoch: u64) -> Vec { let batch = EpochBatch { epoch, txns: vec![SequencedTxn { @@ -689,9 +694,10 @@ pub(super) mod tests { } assert_eq!(seed, 3); - // Seeding is once-only: later ticks must not re-derive and walk backwards - // over epochs this leader has already proposed. - harness.service.current_epoch = Some(9); + // Seeding is once per term: later ticks in the same term must not + // re-derive and walk backwards over epochs this leader already proposed. + let cursor = harness.service.current_epoch.expect("seeded"); + harness.service.current_epoch = Some(EpochCursor { next: 9, ..cursor }); assert_eq!(harness.service.ensure_epoch_seeded(), Some(9)); } @@ -945,6 +951,13 @@ pub(super) mod tests { .inbox .submit_with(make_rw_class(&col_b, &col_a), ®istry) .expect("submit"); + let term = harness + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .leader_term(SEQUENCER_GROUP_ID) + .expect("leader"); + harness.service.current_epoch = Some(EpochCursor { term, next: 4 }); harness.service.mint_epoch(4); @@ -952,7 +965,7 @@ pub(super) mod tests { // Two participants: the vShard it writes and the vShard it reads. assert_eq!(winner.try_recv(), Ok((4, 0, 2))); assert_eq!(registry.pending_assignments_len(), 0); - assert_eq!(harness.service.current_epoch, Some(5)); + assert_eq!(harness.service.current_epoch.map(|c| c.next), Some(5)); } /// A batch whose proposal failed is not in the log. Its callers must fail diff --git a/nodedb-cluster/src/calvin/sequencer/service/epoch_mint.rs b/nodedb-cluster/src/calvin/sequencer/service/epoch_mint.rs index a630c9ad8..5a8446a7f 100644 --- a/nodedb-cluster/src/calvin/sequencer/service/epoch_mint.rs +++ b/nodedb-cluster/src/calvin/sequencer/service/epoch_mint.rs @@ -84,7 +84,7 @@ impl SequencerService { rejected = rejected_count, "epoch tick: all candidates rejected, no proposal" ); - self.current_epoch = Some(epoch + 1); + self.advance_epoch(epoch + 1); return; } @@ -147,7 +147,7 @@ impl SequencerService { rejected = rejected_count, "sequencer proposed epoch batch" ); - self.current_epoch = Some(epoch + 1); + self.advance_epoch(epoch + 1); } Err(e) => { warn!( diff --git a/nodedb-cluster/src/calvin/sequencer/service/epoch_seed.rs b/nodedb-cluster/src/calvin/sequencer/service/epoch_seed.rs index ee9f84b8f..448043cbf 100644 --- a/nodedb-cluster/src/calvin/sequencer/service/epoch_seed.rs +++ b/nodedb-cluster/src/calvin/sequencer/service/epoch_seed.rs @@ -17,8 +17,20 @@ use crate::multi_raft::MultiRaft; use super::core::SequencerService; +/// The next epoch this leader mints, and the sequencer term it was seeded +/// in. +/// +/// The cursor is valid only within its term. Another leader can mint epochs +/// in a later term, so a node that leads again re-derives its seed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct EpochCursor { + pub term: u64, + pub next: u64, +} + impl SequencerService { - /// Derive the epoch seed once, then reuse it for the life of this service. + /// Derive the epoch seed once per sequencer term, then reuse it within + /// that term. /// /// Delegates the safety gate to [`derive_epoch_seed`]. `None` means it is /// not yet safe to mint an epoch on this node and the caller must skip @@ -43,12 +55,33 @@ impl SequencerService { if self.state_machine_halted() { return None; } - if let Some(epoch) = self.current_epoch { - return Some(epoch); + // A cursor from an earlier term can trail epochs a later leader + // minted. Minting from it would send the epoch backwards. + let term = self.sequencer_current_term(); + if let Some(cursor) = self.current_epoch + && cursor.term == term + { + return Some(cursor.next); + } + self.current_epoch = None; + let cursor = derive_epoch_seed(self.node_id, &self.multi_raft, &self.state_machine)?; + self.current_epoch = Some(cursor); + Some(cursor.next) + } + + /// Record that `next` is the next epoch to mint in the cursor's term. + /// No-op without a seeded cursor. + pub(super) fn advance_epoch(&mut self, next: u64) { + if let Some(cursor) = &mut self.current_epoch { + cursor.next = next; } - let epoch = derive_epoch_seed(self.node_id, &self.multi_raft, &self.state_machine)?; - self.current_epoch = Some(epoch); - Some(epoch) + } + + /// This node's current term in the sequencer group, `0` when the group + /// is not mounted here. + fn sequencer_current_term(&self) -> u64 { + let mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + mr.group_leader_at_term(SEQUENCER_GROUP_ID).1 } /// Whether this node's sequencer state machine has stopped applying epoch @@ -91,8 +124,8 @@ impl SequencerService { } } -/// Derive — once — the first epoch this node may propose, returning `None` -/// while it is not yet safe to derive one. +/// Derive the first epoch this node may propose in its current sequencer +/// term, returning `None` while it is not yet safe to derive one. /// /// INVARIANT: **the first epoch a restarted leader proposes must be /// strictly greater than any epoch already committed to the sequencer @@ -125,24 +158,34 @@ impl SequencerService { /// the only thing that puts the first entry in the sequencer log is this /// node proposing under the very seed it is waiting for. /// -/// Returns `None` only while a replay is genuinely in flight; the caller then -/// defers minting for that tick (and only minting — every leader duty that -/// stamps no new identity still runs), so submissions stay queued rather than -/// being sequenced under a colliding epoch. +/// The state machine's history origin decides whether the seed is exact. +/// A state built from the log start, or restored from a snapshot of complete +/// history, holds every committed epoch. Its next epoch is exact, including +/// the first epoch when none committed. A state whose log start was +/// discarded with no snapshot holds only the retained entries. Epochs rise +/// along the log, so a retained epoch still bounds every discarded one. +/// With no retained epoch there is nothing to derive from, and the seed +/// refuses. +/// +/// Returns `None` while a replay is in flight or the history is unknown with +/// no retained epoch. The caller then defers minting for that tick (and only +/// minting — every leader duty that stamps no new identity still runs), so +/// submissions stay queued rather than being sequenced under a colliding +/// epoch. pub(super) fn derive_epoch_seed( node_id: u64, multi_raft: &Mutex, state_machine: &Mutex, -) -> Option { +) -> Option { // Read the Raft-side watermarks and release the lock before taking the // state machine's: the two are never held together anywhere, and this // is the only site that needs both. - let (last_applied, log_tip, first_available) = { + let (last_applied, log_tip, term) = { let mr = multi_raft.lock().unwrap_or_else(|p| p.into_inner()); ( mr.last_applied(SEQUENCER_GROUP_ID), mr.last_log_index(SEQUENCER_GROUP_ID), - mr.first_available_index(SEQUENCER_GROUP_ID), + mr.group_leader_at_term(SEQUENCER_GROUP_ID).1, ) }; let (Some(last_applied), Some(log_tip)) = (last_applied, log_tip) else { @@ -161,18 +204,16 @@ pub(super) fn derive_epoch_seed( } let state_machine = state_machine.lock().unwrap_or_else(|p| p.into_inner()); - // The retained log is the only record of committed epochs. If it starts - // above the first index, earlier entries were discarded (compaction or - // a snapshot install); when none of the retained ones carried an epoch - // there is nothing left to derive a seed from, and minting 0 would - // collide with the discarded history. Refusing to propose is a visible - // stall; minting anyway is silent loss of every batch that follows. - if first_available.unwrap_or(1) > 1 && state_machine.last_applied_epoch().is_none() { + // Refusing to propose is a visible stall. Minting 0 over discarded + // history is silent loss of every batch that follows. + let history = state_machine.history_origin(); + if !history.is_known() && state_machine.last_applied_epoch().is_none() { warn!( node_id, - first_available, - "sequencer log was truncated below every retained epoch; refusing to \ - propose rather than mint an epoch that may collide with discarded history" + ?history, + "sequencer history before the retained log is unknown and no retained \ + entry carries an epoch; refusing to propose rather than mint an epoch \ + that may collide with discarded history" ); return None; } @@ -181,7 +222,129 @@ pub(super) fn derive_epoch_seed( info!( node_id, - epoch, log_tip, "sequencer epoch seed derived from the replayed sequencer log" + epoch, + log_tip, + term, + ?history, + "sequencer epoch seed derived from the replayed sequencer log" ); - Some(epoch) + Some(EpochCursor { term, next: epoch }) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use crate::calvin::CalvinCompletionRegistry; + use crate::calvin::sequencer::config::SEQUENCER_GROUP_ID; + use crate::calvin::sequencer::entry::SequencerEntry; + use crate::calvin::sequencer::service::core::tests::{elect, epoch_batch_bytes, make_harness}; + use crate::calvin::sequencer::state_machine::{HistoryOrigin, SequencerStateMachine}; + + fn detached() -> SequencerStateMachine { + SequencerStateMachine::new(HashMap::new(), CalvinCompletionRegistry::new_detached()) + } + + /// A snapshot taken before any epoch committed holds complete history. + /// The leader that restored it mints the first epoch. + #[test] + fn seed_after_an_epoch_free_snapshot_mints_the_first_epoch() { + let mut harness = make_harness(); + let snapshot = detached().capture_snapshot(5); + { + let mut sm = harness + .state_machine + .lock() + .unwrap_or_else(|p| p.into_inner()); + sm.restore_snapshot(snapshot); + assert_eq!(sm.last_applied_epoch(), None); + assert_eq!(sm.history_origin(), HistoryOrigin::Snapshot { through: 5 }); + } + assert_eq!(harness.service.ensure_epoch_seeded(), Some(0)); + } + + /// A snapshot that holds epoch `n` seeds epoch `n + 1`. + #[test] + fn seed_after_a_snapshot_with_an_epoch_mints_the_next_epoch() { + let mut harness = make_harness(); + let mut leader = detached(); + for epoch in 0..5u64 { + leader.apply(epoch + 1, &epoch_batch_bytes(epoch)); + } + let snapshot = leader.capture_snapshot(5); + harness + .state_machine + .lock() + .unwrap_or_else(|p| p.into_inner()) + .restore_snapshot(snapshot); + assert_eq!(harness.service.ensure_epoch_seeded(), Some(5)); + } + + /// Unknown history with no retained epoch refuses the seed. A retained + /// epoch bounds every discarded one, so it seeds above it. + #[test] + fn unknown_history_without_a_retained_epoch_refuses_the_seed() { + let mut harness = make_harness(); + harness + .state_machine + .lock() + .unwrap_or_else(|p| p.into_inner()) + .mark_history_unknown(); + assert_eq!(harness.service.ensure_epoch_seeded(), None); + assert!(harness.service.current_epoch.is_none()); + + let floor = zerompk::to_msgpack_vec(&SequencerEntry::EpochFloor { + next_epoch: 8, + epoch_system_ms: 0, + }) + .expect("encode"); + harness + .state_machine + .lock() + .unwrap_or_else(|p| p.into_inner()) + .apply(1, &floor); + assert_eq!(harness.service.ensure_epoch_seeded(), Some(8)); + } + + /// A cursor seeded in an earlier term trails the epochs another leader + /// minted since. A new term derives the seed again. + #[test] + fn a_term_change_re_derives_the_seed() { + let mut harness = make_harness(); + assert_eq!(harness.service.ensure_epoch_seeded(), Some(0)); + let first_term = harness.service.current_epoch.expect("seeded").term; + + // Another leader minted epochs 0 and 1, applied here. + { + let mut sm = harness + .state_machine + .lock() + .unwrap_or_else(|p| p.into_inner()); + sm.apply(1, &epoch_batch_bytes(0)); + sm.apply(2, &epoch_batch_bytes(1)); + } + elect(&harness.multi_raft); + { + let mut mr = harness.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + let tip = mr + .last_log_index(SEQUENCER_GROUP_ID) + .expect("group is mounted"); + mr.advance_applied(SEQUENCER_GROUP_ID, tip) + .expect("advance applied"); + } + + assert_eq!(harness.service.ensure_epoch_seeded(), Some(2)); + let cursor = harness.service.current_epoch.expect("seeded"); + assert!(cursor.term > first_term); + } + + /// A tick that finds this node not leading drops the cursor. + #[test] + fn losing_leadership_drops_the_epoch_cursor() { + let mut harness = make_harness(); + assert_eq!(harness.service.ensure_epoch_seeded(), Some(0)); + assert!(harness.service.current_epoch.is_some()); + harness.service.tick(); + assert!(harness.service.current_epoch.is_none()); + } } diff --git a/nodedb-cluster/src/calvin/sequencer/state_machine/apply.rs b/nodedb-cluster/src/calvin/sequencer/state_machine/apply.rs index 94d5d3dae..64c413186 100644 --- a/nodedb-cluster/src/calvin/sequencer/state_machine/apply.rs +++ b/nodedb-cluster/src/calvin/sequencer/state_machine/apply.rs @@ -99,21 +99,6 @@ impl SequencerStateMachine { from_node, result, ), - // Broadcast the OLLP predicate-mismatch signal to ALL replicas so the - // coordinator's registry fires wherever it lives (including remote nodes). - SequencerEntry::OllpMismatch { epoch, position } => self - .completion_registry - .note_ollp_mismatch(TxnId::new(epoch, position)), - // Broadcast the terminal routing-failure signal to ALL replicas so - // the coordinator's registry fires wherever it lives (including - // remote nodes), mirroring `OllpMismatch`. - SequencerEntry::TxnRoutingFailed { - epoch, - position, - detail, - } => self - .completion_registry - .note_routing_failed(TxnId::new(epoch, position), detail), // Durable per-participant votes for a staged cross-shard txn. The // registry tallies them per vshard. Once every participant voted, // the leader aggregates them into the global verdict that gates the @@ -939,33 +924,6 @@ mod tests { assert_eq!(sm.next_epoch(), 1); } - #[tokio::test] - async fn apply_txn_routing_failed_dispatches_to_completion_registry() { - let registry = CalvinCompletionRegistry::new_detached(); - let mut sm = SequencerStateMachine::new(HashMap::new(), Arc::clone(®istry)); - - let data = encode_entry(&SequencerEntry::TxnRoutingFailed { - epoch: 5, - position: 2, - detail: "unroutable plan".to_owned(), - }); - sm.apply(1, &data); - - // The registry's waiter (registered AFTER apply) must still observe - // the failure — `note_routing_failed` persists it on the entry. - let rx = registry.register_completion(crate::calvin::TxnId::new(5, 2), 1); - let outcome = rx.await.expect("routing failure fires"); - assert_eq!( - outcome, - crate::calvin::AttemptOutcome::Failed { - detail: "unroutable plan".to_owned() - } - ); - // TxnRoutingFailed is not an EpochBatch, so it must not perturb the - // epoch counter (mirrors OllpMismatch's non-effect on last_applied_epoch). - assert_eq!(sm.last_applied_epoch(), None); - } - #[tokio::test] async fn apply_verdict_stores_decision_without_perturbing_epoch() { let registry = CalvinCompletionRegistry::new_detached(); @@ -980,8 +938,7 @@ mod tests { // The verdict is stored authoritatively on every replica. assert_eq!(registry.verdict(txn), Some(true)); - // Verdict is not an EpochBatch, so it must not perturb the epoch counter - // (mirrors OllpMismatch/TxnRoutingFailed's non-effect). + // Verdict is not an EpochBatch, so it must not perturb the epoch counter. assert_eq!(sm.last_applied_epoch(), None); } diff --git a/nodedb-cluster/src/calvin/sequencer/state_machine/core.rs b/nodedb-cluster/src/calvin/sequencer/state_machine/core.rs index 64a252010..01698afda 100644 --- a/nodedb-cluster/src/calvin/sequencer/state_machine/core.rs +++ b/nodedb-cluster/src/calvin/sequencer/state_machine/core.rs @@ -102,6 +102,8 @@ pub struct SequencerStateMachine { /// `Txn` inputs this node's schedulers received and may not have made /// durable (see [`super::undurable`]). Local to this replica. pub(super) undurable: super::undurable::UndurableInputs, + /// Where the history this state holds begins (see [`super::history`]). + pub(super) history: super::history::HistoryOrigin, } pub(super) const NOT_YET_APPLIED: u64 = u64::MAX; @@ -151,6 +153,7 @@ impl SequencerStateMachine { completion_registry, open_parts: super::parts::OpenParts::default(), undurable: super::undurable::UndurableInputs::default(), + history: super::history::HistoryOrigin::LogStart, } } diff --git a/nodedb-cluster/src/calvin/sequencer/state_machine/history.rs b/nodedb-cluster/src/calvin/sequencer/state_machine/history.rs new file mode 100644 index 000000000..18750c01c --- /dev/null +++ b/nodedb-cluster/src/calvin/sequencer/state_machine/history.rs @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Where the history the sequencer state machine holds begins. +//! +//! A leader seeds its first epoch from the state machine. The seed is exact +//! only when the state holds the effect of every committed entry. That holds +//! for a state built from the log's first entry, and for one restored from a +//! snapshot that itself held complete history. It does not hold for a log +//! whose start was discarded with no snapshot to restore the state from. + +use super::core::SequencerStateMachine; + +/// Where the state this replica holds begins. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum HistoryOrigin { + /// Built by applying the sequencer log from its first entry. + #[default] + LogStart, + /// Restored from a snapshot that holds the state through log index + /// `through`. Entries after it apply on top. + Snapshot { through: u64 }, + /// The log starts above its first entry, and no snapshot restored the + /// state the discarded entries built. + Unknown, +} + +impl HistoryOrigin { + /// Whether the state holds the effect of every committed entry. + pub fn is_known(self) -> bool { + !matches!(self, Self::Unknown) + } +} + +impl SequencerStateMachine { + /// Where the history this state machine holds begins. + pub fn history_origin(&self) -> HistoryOrigin { + self.history + } + + /// Record that the sequencer log starts above its first entry and no + /// snapshot restored the discarded state. A later snapshot restore + /// replaces it. + pub fn mark_history_unknown(&mut self) { + self.history = HistoryOrigin::Unknown; + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + use crate::calvin::CalvinCompletionRegistry; + + #[test] + fn a_fresh_state_machine_holds_history_from_the_log_start() { + let mut sm = + SequencerStateMachine::new(HashMap::new(), CalvinCompletionRegistry::new_detached()); + assert_eq!(sm.history_origin(), HistoryOrigin::LogStart); + assert!(sm.history_origin().is_known()); + sm.mark_history_unknown(); + assert_eq!(sm.history_origin(), HistoryOrigin::Unknown); + assert!(!sm.history_origin().is_known()); + } +} diff --git a/nodedb-cluster/src/calvin/sequencer/state_machine/mod.rs b/nodedb-cluster/src/calvin/sequencer/state_machine/mod.rs index d6cf02875..47ff562ba 100644 --- a/nodedb-cluster/src/calvin/sequencer/state_machine/mod.rs +++ b/nodedb-cluster/src/calvin/sequencer/state_machine/mod.rs @@ -3,6 +3,7 @@ pub mod apply; pub mod core; pub mod counters; +pub mod history; pub mod parts; pub mod restore; pub mod snapshot; @@ -14,4 +15,5 @@ pub use self::core::{ CutInstantHook, RestorePointHook, SequencerRestorePoint, SequencerStateMachine, }; pub use counters::StateMachineMetrics; +pub use history::HistoryOrigin; pub use snapshot::SequencerSnapshot; diff --git a/nodedb-cluster/src/calvin/sequencer/state_machine/snapshot.rs b/nodedb-cluster/src/calvin/sequencer/state_machine/snapshot.rs index 8b63c1fd3..21ea5fb62 100644 --- a/nodedb-cluster/src/calvin/sequencer/state_machine/snapshot.rs +++ b/nodedb-cluster/src/calvin/sequencer/state_machine/snapshot.rs @@ -10,7 +10,9 @@ //! mints an epoch or an instant at or below a committed one; //! - every open multi-part transaction, with the parts already applied, so //! the follower closes it on the same entry as every other replica, and -//! never abandons one the others committed. +//! never abandons one the others committed; +//! - whether the captured state holds every committed entry's effect, so a +//! follower seeds its epoch from it only when it does. //! //! The leader captures the snapshot at its applied index, on the Raft tick //! thread between apply batches, so it holds exactly the entries through @@ -20,6 +22,7 @@ use crate::calvin::TxnId; use crate::error::ClusterError; use super::core::{NOT_YET_APPLIED, SequencerStateMachine}; +use super::history::HistoryOrigin; /// The state the sequencer log built through `applied_index`. #[derive(Debug, Clone, PartialEq, Eq, zerompk::ToMessagePack, zerompk::FromMessagePack)] @@ -29,6 +32,7 @@ pub struct SequencerSnapshot { last_applied_epoch: Option, last_epoch_system_ms: Option, open_txns: Vec, + history_known: bool, } /// One open multi-part transaction in a [`SequencerSnapshot`]. @@ -83,16 +87,27 @@ impl SequencerStateMachine { last_applied_epoch: self.last_applied_epoch(), last_epoch_system_ms: self.last_epoch_system_ms, open_txns: self.open_parts.images(), + history_known: self.history.is_known(), } } /// Replace the state the log built with `snapshot`, installed as the /// group's log boundary. Entries after the snapshot apply on top. + /// + /// The history origin becomes the snapshot, or stays unknown when the + /// capturing replica's history was unknown. pub fn restore_snapshot(&mut self, snapshot: SequencerSnapshot) { self.last_applied_epoch = snapshot.last_applied_epoch.unwrap_or(NOT_YET_APPLIED); self.last_epoch_system_ms = snapshot.last_epoch_system_ms; self.last_committed_index = snapshot.applied_index; self.open_parts = super::parts::OpenParts::from_images(snapshot.open_txns); + self.history = if snapshot.history_known { + HistoryOrigin::Snapshot { + through: snapshot.applied_index, + } + } else { + HistoryOrigin::Unknown + }; } } @@ -211,6 +226,10 @@ mod tests { assert_eq!(follower.min_open_parts_index(), Some(1)); assert_eq!(follower.last_applied_epoch(), Some(0)); assert_eq!(follower.current_committed_index(), Some(3)); + assert_eq!( + follower.history_origin(), + HistoryOrigin::Snapshot { through: 3 } + ); follower.apply(4, &encode(&part(1, vshard))); assert!( @@ -222,4 +241,19 @@ mod tests { Ok(SchedulerInput::TxnPart { index: 1, .. }) )); } + + /// A snapshot captured with unknown history restores as unknown, so the + /// follower never treats it as complete. + #[test] + fn a_snapshot_of_unknown_history_restores_as_unknown() { + let mut leader = + SequencerStateMachine::new(HashMap::new(), CalvinCompletionRegistry::new_detached()); + leader.mark_history_unknown(); + let snapshot = leader.capture_snapshot(7); + + let mut follower = + SequencerStateMachine::new(HashMap::new(), CalvinCompletionRegistry::new_detached()); + follower.restore_snapshot(snapshot); + assert_eq!(follower.history_origin(), HistoryOrigin::Unknown); + } } diff --git a/nodedb-cluster/src/multi_raft/proposals.rs b/nodedb-cluster/src/multi_raft/proposals.rs index 6c5e4ef0e..8b55bca80 100644 --- a/nodedb-cluster/src/multi_raft/proposals.rs +++ b/nodedb-cluster/src/multi_raft/proposals.rs @@ -207,10 +207,10 @@ impl MultiRaft { return Ok(false); }; // A sequencer replica that installs a snapshot never receives the - // Calvin inputs the snapshot covers, and its schedulers lose them. - // When every voter does, the inputs are gone from the cluster. So no - // replica compacts past an entry a voter's log may lack: no voter - // ever takes a sequencer snapshot, under this leader or a later one. + // Calvin inputs the snapshot covers. So no replica compacts past an + // entry a live voter's log may lack. A voter silent past the + // check-quorum window does not hold the floor. It recovers through a + // sequencer snapshot install once it answers again. let applied_index = if group_id == crate::calvin::SEQUENCER_GROUP_ID { applied_index.min(node.replicated_floor()) } else { diff --git a/nodedb-cluster/src/raft_loop/handle_rpc/consensus.rs b/nodedb-cluster/src/raft_loop/handle_rpc/consensus.rs index b427e057a..f17d8c40a 100644 --- a/nodedb-cluster/src/raft_loop/handle_rpc/consensus.rs +++ b/nodedb-cluster/src/raft_loop/handle_rpc/consensus.rs @@ -12,7 +12,6 @@ use nodedb_raft::message::{ }; use super::super::loop_core::{CommitApplier, RaftLoop}; -use super::membership::TOPOLOGY_GROUP_ID; impl RaftLoop { /// The leader counts a successful answer as this node holding the @@ -165,9 +164,9 @@ impl RaftLoop { { Ok(crate::install_snapshot::ChunkOutcome::Committed(committed)) => { // The watcher means "state visible through N". It moves - // only when the host state machine holds the snapshot. - // Data-group watchers are bumped by the host apply loop. - if group_id == TOPOLOGY_GROUP_ID && committed.state_installed { + // only when the host state machine holds the snapshot, + // for every group kind. Later entries move it on apply. + if committed.state_installed { self.group_watchers.bump(group_id, last_included_index); } // The leader resumes replication after the snapshot index diff --git a/nodedb-cluster/src/raft_loop/tick/apply_committed.rs b/nodedb-cluster/src/raft_loop/tick/apply_committed.rs index 5a7121b78..1cef50fac 100644 --- a/nodedb-cluster/src/raft_loop/tick/apply_committed.rs +++ b/nodedb-cluster/src/raft_loop/tick/apply_committed.rs @@ -202,29 +202,17 @@ impl RaftLoop { if let Err(e) = mr.advance_applied(group_id, last_applied) { warn!(group_id, error = %e, "failed to advance applied index"); } else if group_id == crate::calvin::SEQUENCER_GROUP_ID { - // Sequencer group: the host applies each - // entry to the sequencer state machine - // inline, before returning. The watcher - // is the sequencer's applied index that + // Sequencer group: the host applies each entry to the + // sequencer state machine inline, before returning. The + // watcher is the sequencer's applied index that // authorization lease coverage reports. // - // Data groups are NOT bumped here — for - // them `applier.apply_committed` only - // enqueues entries onto the - // `DistributedApplier` channel; the - // actual data lands in storage when - // `run_apply_loop` finishes the - // SPSC round-trip to the Data Plane. - // The host crate bumps the watcher - // there, so the watermark always means - // "data visible on this node up to - // index N" regardless of which group. - // - // Snapshot-install path also bumps - // (in `super::handle_rpc`) — covers - // jump-on-snapshot for both group - // kinds. The metadata lane bumps the - // metadata group's watcher. + // Data groups are not bumped here. `apply_committed` only + // hands their entries to the host apply loop, which bumps + // the watcher once the data is visible on this node. The + // metadata lane bumps the metadata group's watcher. A + // snapshot install bumps every group whose state it + // restored (see `super::super::handle_rpc`). self.group_watchers.bump(group_id, last_applied); } } diff --git a/nodedb-raft/src/node/core.rs b/nodedb-raft/src/node/core.rs index c31da6293..6485f7c30 100644 --- a/nodedb-raft/src/node/core.rs +++ b/nodedb-raft/src/node/core.rs @@ -116,6 +116,9 @@ pub struct RaftNode { /// Per-voter highest lease round acknowledged in the current term, and /// when the latest answer arrived. Cleared on step-down. pub(super) quorum_window: Vec, + /// When this node won its current leadership term. `None` off the + /// leader path. A voter not yet heard this term counts as last seen here. + pub(super) leader_since: Option, /// Leader-lease rounds and anchor (see [`super::leader_lease`]). pub(super) lease: LeaseState, /// Until this instant the node refuses every vote but a transfer vote. @@ -178,6 +181,7 @@ impl RaftNode { leader_contact: None, last_quorum_contact: None, quorum_window: Vec::new(), + leader_since: None, lease: LeaseState::new(), boot_vote_fence: now + config.election_timeout_max, compaction_ceiling: None, diff --git a/nodedb-raft/src/node/internal.rs b/nodedb-raft/src/node/internal.rs index 978c54400..6c74e3655 100644 --- a/nodedb-raft/src/node/internal.rs +++ b/nodedb-raft/src/node/internal.rs @@ -147,6 +147,7 @@ impl RaftNode { // The contact window belongs to a leader term; it means nothing here. self.last_quorum_contact = None; self.quorum_window.clear(); + self.leader_since = None; self.persist_hard_state(); self.reset_election_timeout(); diff --git a/nodedb-raft/src/node/peer_contact.rs b/nodedb-raft/src/node/peer_contact.rs index 61d986374..c8dcfe5bf 100644 --- a/nodedb-raft/src/node/peer_contact.rs +++ b/nodedb-raft/src/node/peer_contact.rs @@ -18,29 +18,43 @@ impl RaftNode { .map(|leader| leader.ack_count_for(peer)) } - /// The highest log index every voter's log is known to hold. A log - /// compacted at or below it never sends a voter a snapshot, under this - /// leader or a later one: every voter holds the compacted entries. + /// The highest log index every live voter's log is known to hold. See + /// [`Self::replicated_floor_at`]. + pub fn replicated_floor(&self) -> u64 { + self.replicated_floor_at(Instant::now()) + } + + /// The highest log index every live voter's log is known to hold at + /// `now`. A log compacted at or below it never sends a live voter a + /// snapshot. /// /// The leader takes the lowest `match_index` over the other voters, /// capped at its commit index: a committed entry stays in every log that /// holds it. It sends the value with each `AppendEntries`. Every replica - /// keeps the highest value it saw, so the floor never moves back. A voter - /// that stops answering holds it where it is. Learners and observers do - /// not count: a new replica takes a snapshot. - pub fn replicated_floor(&self) -> u64 { + /// keeps the highest value it saw, so the floor never moves back. + /// + /// Only voters that answered within the check-quorum window count. A + /// voter not yet heard this term counts until the window passes from the + /// election win. A silent voter never holds compaction on any replica. + /// This assumes a voter that falls behind the compacted boundary recovers + /// through `InstallSnapshot` once it answers again. + /// + /// Learners and observers never count: a new replica takes a snapshot. + /// Off the leader path this is the highest floor a leader sent. + pub fn replicated_floor_at(&self, now: Instant) -> u64 { let Some(leader) = self.leader_state.as_ref() else { return self.replicated_floor; }; - let lowest_voter = self + let lowest_live_voter = self .config .peers .iter() + .filter(|&&peer| self.voter_heard_within_window(peer, now)) .map(|&peer| leader.match_index_for(peer)) .min() .unwrap_or(u64::MAX); self.replicated_floor - .max(lowest_voter.min(self.volatile.commit_index)) + .max(lowest_live_voter.min(self.volatile.commit_index)) } /// Whether `peer`'s latest response to this leader asked for a snapshot. @@ -73,7 +87,7 @@ impl RaftNode { #[cfg(test)] mod tests { - use std::time::Duration; + use std::time::{Duration, Instant}; use crate::message::{AppendEntriesRequest, RequestVoteResponse}; use crate::node::core::RaftNode; @@ -143,8 +157,9 @@ mod tests { assert_eq!(node.live_leader(std::time::Instant::now()), 0); } - /// The leader's floor is the lowest voter's committed match. A voter - /// that has not answered holds it at 0. + /// The leader's floor is the lowest voter's committed match. Within the + /// window from the election win, a voter that has not answered holds it + /// at 0. #[test] fn the_replicated_floor_waits_for_every_voter() { let mut node = RaftNode::new(test_config(1, vec![2, 3]), MemStorage::new()); @@ -172,6 +187,106 @@ mod tests { assert_eq!(node.replicated_floor(), 1); } + /// Elect node 1 over `cfg`, with every voter granting its vote. + fn elect(cfg: crate::node::config::RaftConfig) -> RaftNode { + let peers = cfg.peers.clone(); + let mut node = RaftNode::new(cfg, MemStorage::new()); + force_election(&mut node); + for peer in peers { + if node.role() == NodeRole::Leader { + break; + } + node.handle_request_vote_response( + peer, + &RequestVoteResponse { + term: node.current_term(), + vote_granted: true, + }, + ); + } + assert_eq!(node.role(), NodeRole::Leader); + let _ = node.take_ready(); + node + } + + /// Peer `peer` answers the latest round, acking `last_log_index`. + fn answer(node: &mut RaftNode, peer: u64, success: bool, last_log_index: u64) { + let resp = crate::message::AppendEntriesResponse { + term: node.current_term(), + success, + last_log_index, + round: node.lease.next_round - 1, + needs_snapshot: false, + }; + node.handle_append_entries_response(peer, &resp); + } + + /// Move the election win back past the check-quorum window. + fn age_term_start(node: &mut RaftNode) { + let window = node.config.election_timeout_max; + node.leader_since = Some(Instant::now() - window - Duration::from_millis(1)); + } + + /// A voter silent for the whole check-quorum window stops holding the + /// floor. The voter that answered sets it. + #[test] + fn a_silent_voter_stops_holding_the_floor_after_the_window() { + let mut node = elect(test_config(1, vec![2, 3])); + answer(&mut node, 2, true, 1); + assert_eq!(node.commit_index(), 1); + assert_eq!( + node.replicated_floor(), + 0, + "voter 3 counts within the window from the election win" + ); + age_term_start(&mut node); + answer(&mut node, 2, true, 1); + assert_eq!( + node.replicated_floor(), + 1, + "silent voter 3 no longer counts" + ); + } + + /// A voter that answered within the window holds the floor at its + /// `match_index`, even when that answer was a rejection. Voter 3 answers + /// before the commit moves: the floor never moves back. + #[test] + fn a_voter_heard_within_the_window_still_holds_the_floor() { + let mut node = elect(test_config(1, vec![2, 3])); + age_term_start(&mut node); + answer(&mut node, 3, false, 0); + assert!(node.voter_heard_within_window(3, Instant::now())); + answer(&mut node, 2, true, 1); + assert_eq!(node.commit_index(), 1); + assert_eq!( + node.replicated_floor(), + 0, + "voter 3 answered and holds nothing" + ); + + let stale = Instant::now() + node.config.election_timeout_max; + assert!( + !node.voter_heard_within_window(3, stale), + "the answer ages out of the window" + ); + } + + /// A learner never holds the floor, answered or silent. + #[test] + fn learners_never_count_toward_the_floor() { + let mut cfg = test_config(1, vec![2, 3]); + cfg.learners = vec![4]; + let mut node = elect(cfg); + answer(&mut node, 4, false, 0); + answer(&mut node, 2, true, 1); + answer(&mut node, 3, true, 1); + assert_eq!(node.commit_index(), 1); + assert_eq!(node.replicated_floor(), 1, "learner 4 holds nothing"); + age_term_start(&mut node); + assert_eq!(node.replicated_floor(), 1); + } + /// A follower keeps the highest floor a leader sent it. #[test] fn a_follower_keeps_the_highest_replicated_floor() { diff --git a/nodedb-raft/src/node/quorum_contact.rs b/nodedb-raft/src/node/quorum_contact.rs index 2b44ac90d..74f32f7a2 100644 --- a/nodedb-raft/src/node/quorum_contact.rs +++ b/nodedb-raft/src/node/quorum_contact.rs @@ -66,10 +66,24 @@ impl RaftNode { /// grant does not make a voter refuse other candidates. pub(super) fn arm_quorum_window(&mut self, now: Instant) { self.last_quorum_contact = Some(now); + self.leader_since = Some(now); self.quorum_window.clear(); self.lease.begin_term(); } + /// Whether voter `peer` answered this leader within the check-quorum + /// window at `now`. A voter not yet heard this term counts from the + /// election win. Always false off the leader path. + pub(super) fn voter_heard_within_window(&self, peer: u64, now: Instant) -> bool { + let last_seen = match self.voter_ack(peer) { + Some(ack) => Some(ack.arrived), + None => self.leader_since, + }; + last_seen.is_some_and(|seen| { + now.saturating_duration_since(seen) < self.config.election_timeout_max + }) + } + /// Move contact up to the latest instant by which a quorum of voters /// answered this term. No-op for any role but leader. /// diff --git a/nodedb-types/src/config/tuning/calvin.rs b/nodedb-types/src/config/tuning/calvin.rs new file mode 100644 index 000000000..cc98d759f --- /dev/null +++ b/nodedb-types/src/config/tuning/calvin.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Calvin scheduler tuning — channel sizes, deadlines, and the in-flight +//! bounds of each per-vShard scheduler. +//! +//! The epoch duration belongs to the sequencer, so it is not set here. The +//! scheduler reads it from the sequencer config it runs beside. + +use serde::{Deserialize, Serialize}; + +fn default_channel_capacity() -> usize { + 512 +} + +fn default_txn_deadline_multiplier() -> u32 { + 3 +} + +fn default_dependent_read_passive_timeout_ms() -> u64 { + // Three 20 ms sequencer epochs. + 60 +} + +fn default_verdict_stall_warn_ms() -> u64 { + // 250 sequencer epochs of 20 ms. + 5_000 +} + +fn default_max_inflight_backlog() -> usize { + // The per-core request queue capacity of the bridge dispatcher. Every + // dispatch of one vShard goes to one Data Plane core. + 1_024 +} + +fn default_catch_up_window() -> u64 { + // One channel of missed entries per drain. + 512 +} + +/// Tuning knobs for the Calvin scheduler that runs per hosted vShard. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CalvinTuning { + /// Capacity of each scheduler's bounded input, completion, read-result, + /// and verdict channels. + #[serde(default = "default_channel_capacity")] + pub channel_capacity: usize, + + /// Transaction deadline, in sequencer epochs. + #[serde(default = "default_txn_deadline_multiplier")] + pub txn_deadline_multiplier: u32, + + /// Timeout in milliseconds for passive participant read results of a + /// dependent-read transaction. + #[serde(default = "default_dependent_read_passive_timeout_ms")] + pub dependent_read_passive_timeout_ms: u64, + + /// Interval in milliseconds between stall warnings for a transaction + /// parked on its global verdict. The transaction keeps waiting and never + /// aborts. The idle sweep runs at a quarter of this interval. + #[serde(default = "default_verdict_stall_warn_ms")] + pub verdict_stall_warn_ms: u64, + + /// In-flight backlog at which a scheduler stops taking new sequenced + /// input. The backlog counts pending, blocked, and dependent-barrier + /// transactions. + #[serde(default = "default_max_inflight_backlog")] + pub max_inflight_backlog: usize, + + /// Most sequencer log entries one catch-up drain reads and replays. + #[serde(default = "default_catch_up_window")] + pub catch_up_window: u64, +} + +impl Default for CalvinTuning { + fn default() -> Self { + Self { + channel_capacity: default_channel_capacity(), + txn_deadline_multiplier: default_txn_deadline_multiplier(), + dependent_read_passive_timeout_ms: default_dependent_read_passive_timeout_ms(), + verdict_stall_warn_ms: default_verdict_stall_warn_ms(), + max_inflight_backlog: default_max_inflight_backlog(), + catch_up_window: default_catch_up_window(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults() { + let t = CalvinTuning::default(); + assert_eq!(t.channel_capacity, 512); + assert_eq!(t.txn_deadline_multiplier, 3); + assert_eq!(t.dependent_read_passive_timeout_ms, 60); + assert_eq!(t.verdict_stall_warn_ms, 5_000); + assert_eq!(t.max_inflight_backlog, 1_024); + assert_eq!(t.catch_up_window, 512); + } + + #[test] + fn partial_override() { + let toml_str = r#" +max_inflight_backlog = 256 +verdict_stall_warn_ms = 10000 +"#; + let t: CalvinTuning = toml::from_str(toml_str).expect("deserialize"); + assert_eq!(t.max_inflight_backlog, 256); + assert_eq!(t.verdict_stall_warn_ms, 10_000); + assert_eq!(t.channel_capacity, 512); + assert_eq!(t.catch_up_window, 512); + } + + #[test] + fn empty_table_keeps_the_defaults() { + let t: CalvinTuning = toml::from_str("").expect("deserialize"); + assert_eq!(t, CalvinTuning::default()); + } +} diff --git a/nodedb-types/src/config/tuning/config.rs b/nodedb-types/src/config/tuning/config.rs index aac570e7f..b7b690db9 100644 --- a/nodedb-types/src/config/tuning/config.rs +++ b/nodedb-types/src/config/tuning/config.rs @@ -5,6 +5,7 @@ use serde::{Deserialize, Serialize}; use super::bitemporal::BitemporalTuning; +use super::calvin::CalvinTuning; use super::data_plane::{DataPlaneTuning, QueryTuning}; use super::engines::{GraphTuning, KvTuning, SparseTuning, TimeseriesToning, VectorTuning}; use super::maintenance::MaintenanceTuning; @@ -51,6 +52,8 @@ pub struct TuningConfig { pub bitemporal: BitemporalTuning, #[serde(default)] pub maintenance: MaintenanceTuning, + #[serde(default)] + pub calvin: CalvinTuning, } impl TuningConfig { @@ -112,6 +115,18 @@ mod tests { assert_eq!(parsed.memory.doc_cache_entries, 4096); assert_eq!(parsed.shutdown.deadline_ms, 900); assert_eq!(parsed.maintenance.auto_analyze_min_mutations, 1000); + assert_eq!(parsed.calvin, CalvinTuning::default()); + } + + #[test] + fn calvin_section_overrides_the_scheduler_backlog() { + let toml_str = r#" +[calvin] +max_inflight_backlog = 256 +"#; + let cfg: TuningConfig = toml::from_str(toml_str).expect("deserialize"); + assert_eq!(cfg.calvin.max_inflight_backlog, 256); + assert_eq!(cfg.calvin.channel_capacity, 512); } #[test] diff --git a/nodedb-types/src/config/tuning/mod.rs b/nodedb-types/src/config/tuning/mod.rs index d521e851f..445ce2e86 100644 --- a/nodedb-types/src/config/tuning/mod.rs +++ b/nodedb-types/src/config/tuning/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 mod bitemporal; +mod calvin; mod config; mod data_plane; mod engines; @@ -11,6 +12,7 @@ mod scheduler; mod shutdown; pub use bitemporal::BitemporalTuning; +pub use calvin::CalvinTuning; pub use config::TuningConfig; pub use data_plane::{DataPlaneTuning, QueryTuning}; pub use engines::{ diff --git a/nodedb/src/bootstrap/calvin_catch_up.rs b/nodedb/src/bootstrap/calvin_catch_up.rs new file mode 100644 index 000000000..7419b0611 --- /dev/null +++ b/nodedb/src/bootstrap/calvin_catch_up.rs @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Calvin scheduler catch-up gate. +//! +//! Each Calvin scheduler starts with a rebuild target: the highest epoch its +//! recovery found applied for the vShard. Until its fully-applied watermark +//! reaches that target, the vShard re-applies epochs from the sequencer log. +//! A gateway opened meanwhile serves a vShard whose Calvin state is behind +//! what this node acknowledged before the restart. +//! +//! This gate holds startup until every scheduler running on this node reports +//! caught up. A scheduler with nothing to rebuild reports at once, and a node +//! that runs no scheduler passes at once. A vShard whose scheduler halted can +//! never catch up, so its halt fails the wait at once. The wait shares the +//! data-group recovery bound and fails closed when the bound expires. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use tracing::{debug, info}; + +use crate::bootstrap::data_group_recovery::DATA_GROUP_RECOVERY_TIMEOUT; +use crate::control::cluster::CalvinApplyHalt; +use crate::control::state::SharedState; + +/// How often the caught-up registry is re-read while waiting. Catch-up drains +/// run on a scheduler tick of about a second, so a coarse poll costs nothing. +const POLL_INTERVAL: Duration = Duration::from_millis(50); + +/// Most lagging vShards one error message names. The count names the rest. +const MAX_NAMED_VSHARDS: usize = 16; + +/// Why the scheduler catch-up wait cannot pass yet. +#[derive(Debug, Clone, PartialEq, Eq)] +enum CatchUpWait { + /// These vShards' schedulers are behind their rebuild target. + Lagging(Vec), + /// A lagging vShard's scheduler halted, so it will never catch up. + Halted(CalvinApplyHalt), +} + +/// The wait state for `lagging` vShards, given the node's first apply halt. +fn catch_up_wait(lagging: Vec, halt: Option<&CalvinApplyHalt>) -> Option { + if lagging.is_empty() { + return None; + } + if let Some(halt) = halt + && lagging.binary_search(&halt.vshard_id).is_ok() + { + return Some(CatchUpWait::Halted(halt.clone())); + } + Some(CatchUpWait::Lagging(lagging)) +} + +/// Name at most [`MAX_NAMED_VSHARDS`] of `lagging`, then count the rest. +fn describe_lagging(lagging: &[u32]) -> String { + let named = lagging + .iter() + .take(MAX_NAMED_VSHARDS) + .map(u32::to_string) + .collect::>() + .join(", "); + match lagging.len().checked_sub(MAX_NAMED_VSHARDS) { + Some(rest) if rest > 0 => format!("vShards {named} and {rest} more"), + _ => format!("vShards {named}"), + } +} + +/// Hold startup until every Calvin scheduler on this node reaches its rebuild +/// target. +pub async fn await_calvin_catch_up(shared: &Arc) -> anyhow::Result<()> { + let registry = shared.authorization_fence.calvin_mirrors().caught_up(); + let deadline = Instant::now() + DATA_GROUP_RECOVERY_TIMEOUT; + + loop { + let halt = shared.sequencer_halt.apply_halt().report(); + match catch_up_wait(registry.lagging(), halt) { + None => { + info!("every calvin scheduler reached its rebuild target"); + return Ok(()); + } + Some(CatchUpWait::Halted(halt)) => { + return Err(anyhow::anyhow!( + "calvin scheduler for vShard {} halted at epoch {} position {} \ + ({}, step {}): {}; it cannot reach its rebuild target", + halt.vshard_id, + halt.epoch, + halt.position, + halt.reason, + halt.step, + halt.error + )); + } + Some(CatchUpWait::Lagging(lagging)) => { + if Instant::now() >= deadline { + return Err(anyhow::anyhow!( + "calvin scheduler catch-up timeout after \ + {DATA_GROUP_RECOVERY_TIMEOUT:?}: {} behind their rebuild target", + describe_lagging(&lagging) + )); + } + debug!( + lagging = lagging.len(), + "waiting for calvin schedulers to reach their rebuild target" + ); + } + } + tokio::time::sleep(POLL_INTERVAL).await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn halt(vshard_id: u32) -> CalvinApplyHalt { + CalvinApplyHalt { + vshard_id, + epoch: 9, + position: 1, + reason: "flush_failed", + step: "flush", + error: "flush returned Error".to_string(), + } + } + + #[test] + fn no_lagging_scheduler_passes() { + assert_eq!(catch_up_wait(Vec::new(), None), None); + assert_eq!(catch_up_wait(Vec::new(), Some(&halt(3))), None); + } + + #[test] + fn a_lagging_scheduler_waits() { + assert_eq!( + catch_up_wait(vec![2, 5], None), + Some(CatchUpWait::Lagging(vec![2, 5])) + ); + } + + #[test] + fn a_halt_on_another_vshard_keeps_waiting() { + assert_eq!( + catch_up_wait(vec![2, 5], Some(&halt(3))), + Some(CatchUpWait::Lagging(vec![2, 5])) + ); + } + + #[test] + fn a_halted_lagging_scheduler_fails_at_once() { + assert_eq!( + catch_up_wait(vec![2, 5], Some(&halt(5))), + Some(CatchUpWait::Halted(halt(5))) + ); + } + + #[test] + fn a_long_lagging_list_names_the_first_vshards_and_counts_the_rest() { + let lagging: Vec = (0..20).collect(); + let text = describe_lagging(&lagging); + assert!(text.starts_with("vShards 0, 1, 2"), "{text}"); + assert!(text.ends_with("15 and 4 more"), "{text}"); + assert_eq!(describe_lagging(&[7]), "vShards 7"); + } + + /// A node that runs no scheduler, as a fresh single node does before its + /// first sequenced write, passes at once. + #[tokio::test] + async fn a_node_with_no_scheduler_passes_at_once() { + let dir = tempfile::tempdir().expect("tmpdir"); + let wal = Arc::new( + crate::wal::WalManager::open_for_testing(&dir.path().join("catch-up.wal")) + .expect("open wal"), + ); + let (dispatcher, _data_sides) = crate::bridge::Dispatcher::new(1, 64); + let shared = SharedState::new(dispatcher, wal).expect("build shared state"); + await_calvin_catch_up(&shared) + .await + .expect("no scheduler holds the gate"); + } +} diff --git a/nodedb/src/bootstrap/cluster_ready.rs b/nodedb/src/bootstrap/cluster_ready.rs index f35bfabf0..f6b24d6e0 100644 --- a/nodedb/src/bootstrap/cluster_ready.rs +++ b/nodedb/src/bootstrap/cluster_ready.rs @@ -161,6 +161,15 @@ pub async fn await_cluster_ready( return Err(e); } + // A replayed data group says nothing about Calvin: each scheduler still + // re-applies epochs from the sequencer log up to the highest epoch it had + // applied before the restart. Hold the gateway until every scheduler on + // this node reaches that target. Fail closed, like the wait above. + if let Err(e) = crate::bootstrap::calvin_catch_up::await_calvin_catch_up(shared).await { + data_groups_gate.fail(format!("calvin scheduler catch-up failed: {e}")); + return Err(e); + } + // Retry every owed reclaim before the gateway opens. A row that does not // reclaim keeps the drain hold its retry took, so a same-name CREATE waits // for the worker's retry and never opens over storage the retry erases. diff --git a/nodedb/src/bootstrap/mod.rs b/nodedb/src/bootstrap/mod.rs index 005a9adb9..d57dfdff0 100644 --- a/nodedb/src/bootstrap/mod.rs +++ b/nodedb/src/bootstrap/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 pub mod background_loops; +pub mod calvin_catch_up; pub mod catalog_open; pub mod cluster_ready; pub mod constraint_reconcile; diff --git a/nodedb/src/bridge/dispatch/core_channel.rs b/nodedb/src/bridge/dispatch/core_channel.rs index 4f2175c31..9ae4ea45a 100644 --- a/nodedb/src/bridge/dispatch/core_channel.rs +++ b/nodedb/src/bridge/dispatch/core_channel.rs @@ -44,7 +44,7 @@ pub struct CoreChannel { /// **Lifetime**: entries are written in place and never reach a "remove" /// path on their own. Stale databases that no longer enqueue requests /// retain a `Normal` (or last-observed) entry until the surrounding - /// dispatcher is dropped or `recalculate_tenant_limits` rotates state. + /// dispatcher is dropped. /// The map is bounded by the universe of `database_id`s that have ever /// been dispatched against this core, so unbounded growth is not a /// concern in practice. diff --git a/nodedb/src/bridge/dispatch/dispatcher.rs b/nodedb/src/bridge/dispatch/dispatcher.rs index d69296ce3..8847beeb8 100644 --- a/nodedb/src/bridge/dispatch/dispatcher.rs +++ b/nodedb/src/bridge/dispatch/dispatcher.rs @@ -24,6 +24,7 @@ use super::core_channel::{CoreChannel, CoreChannelDataSide}; use super::dispatched_lsns::DispatchedLsns; use super::journal::WriteSetJournal; use super::outcome_floor::OutcomeFloor; +use super::tenant_inflight::TenantInflight; /// Per-core request queue capacity of the server's bridge dispatcher. /// @@ -104,16 +105,11 @@ pub struct Dispatcher { /// Routes vShards to core IDs. pub(super) router: VShardRouter, - /// Per-tenant in-flight request count across all cores. - pub(super) tenant_inflight: HashMap, + /// Per-tenant in-flight counts across all cores and the fair-share cap + /// they derive. + pub(super) tenants: TenantInflight, - /// Maps request_id → tenant_id for in-flight requests. - pub(super) request_tenant: HashMap, - - /// Maximum in-flight requests per tenant (0 = unlimited). - pub(super) max_per_tenant_inflight: u32, - - /// Per-core queue capacity (used in tenant fairness recalculation). + /// Per-core queue capacity. pub(super) per_core_capacity: u32, /// Resolves priority class for a database_id (consulted on enqueue). @@ -184,9 +180,7 @@ impl Dispatcher { Self { cores, router, - tenant_inflight: HashMap::new(), - request_tenant: HashMap::new(), - max_per_tenant_inflight: total_capacity as u32, + tenants: TenantInflight::new(u32::try_from(total_capacity).unwrap_or(u32::MAX)), per_core_capacity: queue_capacity as u32, priority_resolver, data_plane_draining: false, diff --git a/nodedb/src/bridge/dispatch/drain.rs b/nodedb/src/bridge/dispatch/drain.rs index 064ba8155..79dba5960 100644 --- a/nodedb/src/bridge/dispatch/drain.rs +++ b/nodedb/src/bridge/dispatch/drain.rs @@ -27,7 +27,6 @@ use crate::bridge::envelope::{ErrorCode, Payload, Status}; use crate::types::{Lsn, RequestId}; use super::dispatcher::Dispatcher; -use super::enqueue::release_inflight_slot; /// Work one core still owes the Control Plane. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -155,8 +154,7 @@ impl Dispatcher { for rid in ids { // The node is shutting down, and this core publishes nothing more. self.dispatched_lsns.settle(rid); - freed |= - release_inflight_slot(&mut self.request_tenant, &mut self.tenant_inflight, rid); + freed |= self.tenants.release(rid); abandoned.push(envelope::Response { request_id: RequestId::new(rid), status: Status::Error, diff --git a/nodedb/src/bridge/dispatch/enqueue.rs b/nodedb/src/bridge/dispatch/enqueue.rs index ade60a51f..29249e480 100644 --- a/nodedb/src/bridge/dispatch/enqueue.rs +++ b/nodedb/src/bridge/dispatch/enqueue.rs @@ -6,8 +6,6 @@ //! A capacity limit refuses with [`crate::Error::DispatchCapacity`] and hands //! the request back. Every other refusal is terminal. -use std::collections::HashMap; - use tracing::warn; use crate::DispatchCapacityScope; @@ -18,6 +16,7 @@ use crate::types::{Lsn, VShardId}; use super::dispatcher::Dispatcher; use super::journal::JournalGroup; use super::refusal::DispatchRefusal; +use super::tenant_inflight::TenantAtCap; impl Dispatcher { /// Dispatch a request to the correct Data Plane core. @@ -58,19 +57,16 @@ impl Dispatcher { let wal_lsn = request.wal_lsn; // Per-tenant fairness: refuse while the tenant holds its in-flight cap. - if self.max_per_tenant_inflight > 0 { - let inflight = self.tenant_inflight.get(&tenant_id).copied().unwrap_or(0); - if inflight >= self.max_per_tenant_inflight { - let scope = DispatchCapacityScope::TenantInflight { - tenant_id: request.tenant_id, - inflight, - cap: self.max_per_tenant_inflight, - }; - return Err(DispatchRefusal::boxed( - crate::Error::DispatchCapacity { scope }, - request, - )); - } + if let Some(TenantAtCap { inflight, cap }) = self.tenants.at_cap(tenant_id) { + let scope = DispatchCapacityScope::TenantInflight { + tenant_id: request.tenant_id, + inflight, + cap, + }; + return Err(DispatchRefusal::boxed( + crate::Error::DispatchCapacity { scope }, + request, + )); } let Some(core_id) = self.router.resolve(request.vshard_id) else { @@ -167,14 +163,6 @@ impl Dispatcher { Ok(()) } - /// Recalculate the per-tenant in-flight limit based on active tenants. - pub fn recalculate_tenant_limits(&mut self) { - let active = self.tenant_inflight.len().max(1) as u32; - let total_capacity: u32 = self.cores.len() as u32 * self.per_core_capacity; - self.max_per_tenant_inflight = (total_capacity / active).max(2); - self.tenant_inflight.retain(|_, count| *count > 0); - } - /// Bookkeeping once a request sits in `core_id`'s weighted-fair queue: /// hold the outcome floor below its WAL LSN, flush it toward the ring, /// record pressure, track it as outstanding and in flight for its tenant, @@ -217,8 +205,7 @@ impl Dispatcher { channel.outstanding.insert(req_id); // Track per-tenant in-flight + request→tenant mapping for response routing. - *self.tenant_inflight.entry(tenant_id).or_insert(0) += 1; - self.request_tenant.insert(req_id, tenant_id); + self.tenants.admit(tenant_id, req_id); // Wake the Data Plane core via eventfd. if let Some(ref notifier) = self.cores[core_id].wake_notifier { @@ -227,27 +214,6 @@ impl Dispatcher { } } -/// Release the in-flight slot `request_id` holds for its tenant. -/// -/// Returns `true` when a slot was freed. A free function over the two maps, -/// so a caller can release while it holds a borrow of one core's channel. -pub(super) fn release_inflight_slot( - request_tenant: &mut HashMap, - tenant_inflight: &mut HashMap, - request_id: u64, -) -> bool { - let Some(tenant_id) = request_tenant.remove(&request_id) else { - return false; - }; - match tenant_inflight.get_mut(&tenant_id) { - Some(count) if *count > 0 => { - *count -= 1; - true - } - _ => false, - } -} - #[cfg(test)] mod tests { use super::*; @@ -306,6 +272,72 @@ mod tests { ); } + /// Dispatch a request of `tenant` with id and database `id`, then pop it + /// off the ring so the ring and the weighted-fair queue stay empty. + fn dispatch_popped( + dispatcher: &mut Dispatcher, + data_side: &mut crate::bridge::dispatch::CoreChannelDataSide, + tenant: u64, + id: u64, + ) -> crate::Result<()> { + let mut request = make_request_for_db(0, id, id); + request.tenant_id = TenantId::new(tenant); + dispatcher.dispatch(request)?; + while data_side.request_rx.try_pop().is_ok() {} + Ok(()) + } + + fn tenant_cap_of(error: crate::Error) -> u32 { + match error { + crate::Error::DispatchCapacity { + scope: DispatchCapacityScope::TenantInflight { cap, .. }, + } => cap, + other => panic!("expected a tenant-cap refusal, got: {other}"), + } + } + + #[test] + fn two_active_tenants_split_the_capacity_and_an_idle_one_returns_its_share() { + let (mut dispatcher, mut data_sides) = Dispatcher::new(1, 8); + let data_side = &mut data_sides[0]; + + // Tenant 2 holds one request, so tenant 1 gets half of the eight slots. + dispatch_popped(&mut dispatcher, data_side, 2, 100).expect("tenant 2 admitted"); + for id in 1..=4u64 { + dispatch_popped(&mut dispatcher, data_side, 1, id).expect("within half the capacity"); + } + let refused = dispatch_popped(&mut dispatcher, data_side, 1, 5) + .expect_err("tenant 1 is at half the capacity"); + assert_eq!(tenant_cap_of(refused), 4); + + // Tenant 2's only request completes, so tenant 1 is the sole tenant. + data_side + .response_tx + .try_push(BridgeResponse { + inner: envelope::Response { + request_id: RequestId::new(100), + status: Status::Ok, + attempt: 1, + partial: false, + payload: Payload::empty(), + watermark_lsn: Lsn::ZERO, + error_code: None, + read_set_valid: None, + read_version_lsn: Lsn::ZERO, + write_set: Vec::new(), + }, + }) + .expect("response ring has room"); + assert_eq!(dispatcher.poll_responses().len(), 1); + + for id in 5..=8u64 { + dispatch_popped(&mut dispatcher, data_side, 1, id).expect("the full capacity is back"); + } + let refused = dispatch_popped(&mut dispatcher, data_side, 1, 9) + .expect_err("tenant 1 is at the full capacity"); + assert_eq!(tenant_cap_of(refused), 8); + } + #[test] fn full_weighted_fair_queue_is_refused_with_queue_full_scope() { // Distinct tenants and databases keep the tenant cap and the per-DB @@ -339,8 +371,8 @@ mod tests { dispatcher.dispatch_to_core(1, request).unwrap(); - assert_eq!(dispatcher.tenant_inflight.get(&tenant_id), Some(&1)); - assert_eq!(dispatcher.request_tenant.get(&request_id), Some(&tenant_id)); + assert_eq!(dispatcher.tenants.inflight(tenant_id), 1); + assert_eq!(dispatcher.tenants.tenant_of(request_id), Some(tenant_id)); assert_eq!(data_sides[1].request_rx.len(), 1); let _req = data_sides[1].request_rx.try_pop().unwrap(); @@ -364,8 +396,8 @@ mod tests { let responses = dispatcher.poll_responses(); assert_eq!(responses.len(), 1); - assert_eq!(dispatcher.tenant_inflight.get(&tenant_id), Some(&0)); - assert!(!dispatcher.request_tenant.contains_key(&request_id)); + assert_eq!(dispatcher.tenants.inflight(tenant_id), 0); + assert_eq!(dispatcher.tenants.tenant_of(request_id), None); } #[test] diff --git a/nodedb/src/bridge/dispatch/mod.rs b/nodedb/src/bridge/dispatch/mod.rs index dbfa33695..ce643e371 100644 --- a/nodedb/src/bridge/dispatch/mod.rs +++ b/nodedb/src/bridge/dispatch/mod.rs @@ -10,6 +10,7 @@ mod journal; mod outcome_floor; mod refusal; mod response_poll; +mod tenant_inflight; #[cfg(test)] mod test_requests; diff --git a/nodedb/src/bridge/dispatch/response_poll.rs b/nodedb/src/bridge/dispatch/response_poll.rs index a999c5007..5eba45dbc 100644 --- a/nodedb/src/bridge/dispatch/response_poll.rs +++ b/nodedb/src/bridge/dispatch/response_poll.rs @@ -10,7 +10,6 @@ use crate::bridge::envelope::{ErrorCode, Payload, Status}; use crate::types::{Lsn, RequestId}; use super::dispatcher::Dispatcher; -use super::enqueue::release_inflight_slot; impl Dispatcher { /// Poll responses from all Data Plane cores. @@ -21,7 +20,7 @@ impl Dispatcher { /// everything dispatched earlier that it never answered. Those travel back /// with the real responses so the single completion loop in the caller /// finishes each waiter, and the loop below releases each request's - /// `tenant_inflight` slot exactly as a real response would — without which + /// tenant in-flight slot exactly as a real response would — without which /// one dead core ratchets the tenant's in-flight count until the tenant is /// rejected on healthy cores too. /// @@ -43,11 +42,7 @@ impl Dispatcher { if !br.inner.partial { channel.outstanding.remove(&rid); self.dispatched_lsns.settle(rid); - freed |= release_inflight_slot( - &mut self.request_tenant, - &mut self.tenant_inflight, - rid, - ); + freed |= self.tenants.release(rid); } responses.push(br.inner); } @@ -85,8 +80,7 @@ impl Dispatcher { for rid in lost { // A dead core never publishes a watermark again. self.dispatched_lsns.settle(rid); - freed |= - release_inflight_slot(&mut self.request_tenant, &mut self.tenant_inflight, rid); + freed |= self.tenants.release(rid); responses.push(envelope::Response { request_id: RequestId::new(rid), status: Status::Error, @@ -196,7 +190,7 @@ mod tests { // When a Data Plane core's consumer/producer is dropped (the core thread // died), `Dispatcher` must synthesize an error `Response` for every // request it knows is outstanding on that core, rather than dropping the - // request silently and leaking the caller's waiter + `tenant_inflight` + // request silently and leaking the caller's waiter + tenant in-flight // slot forever. Dropping one element of the `data_sides` vector handed // back by `Dispatcher::new`/`with_resolver` simulates that core thread // dying, matching how `dispatch_routes_to_correct_core` and @@ -236,7 +230,7 @@ mod tests { #[test] fn dead_core_synthesized_response_resets_tenant_inflight() { - // The ratchet: `tenant_inflight` is incremented on dispatch and must + // The ratchet: the tenant in-flight count is incremented on dispatch and must // return to its pre-dispatch value once the synthesized response for // the lost request is drained through `poll_responses` — otherwise it // climbs forever and eventually starves the tenant on healthy cores. @@ -247,31 +241,23 @@ mod tests { let request = make_request_for_db(0, 1, 1); let tenant_id = request.tenant_id.as_u64(); - let before = dispatcher - .tenant_inflight - .get(&tenant_id) - .copied() - .unwrap_or(0); + let before = dispatcher.tenants.inflight(tenant_id); dispatcher.dispatch_to_core(dead_core, request).unwrap(); assert_eq!( - dispatcher.tenant_inflight.get(&tenant_id).copied(), - Some(before + 1), - "dispatch must still increment tenant_inflight even though the core is dead" + dispatcher.tenants.inflight(tenant_id), + before + 1, + "dispatch must still increment the tenant in-flight count even though the core is dead" ); let responses = dispatcher.poll_responses(); assert_eq!(responses.len(), 1); assert_eq!( - dispatcher - .tenant_inflight - .get(&tenant_id) - .copied() - .unwrap_or(0), + dispatcher.tenants.inflight(tenant_id), before, - "tenant_inflight must return to its pre-dispatch value, not ratchet upward" + "the tenant in-flight count must return to its pre-dispatch value, not ratchet upward" ); - assert!(!dispatcher.request_tenant.contains_key(&1)); + assert_eq!(dispatcher.tenants.tenant_of(1), None); } #[test] diff --git a/nodedb/src/bridge/dispatch/tenant_inflight.rs b/nodedb/src/bridge/dispatch/tenant_inflight.rs new file mode 100644 index 000000000..530318505 --- /dev/null +++ b/nodedb/src/bridge/dispatch/tenant_inflight.rs @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Per-tenant in-flight accounting and the fair-share cap it derives. +//! +//! Each tenant with a request in flight is active. The cap is the dispatcher's +//! total capacity divided by the active tenant count, at least two. The cap +//! moves only when a tenant turns active or idle. A move gates new admissions +//! only: a request already admitted keeps its slot until it is answered. + +use std::collections::HashMap; + +/// Smallest per-tenant cap. One slot will serialize every request a tenant +/// sends, so the floor keeps one request in flight while the next one queues. +const MIN_TENANT_CAP: u32 = 2; + +/// A tenant held at its in-flight cap. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct TenantAtCap { + pub(super) inflight: u32, + pub(super) cap: u32, +} + +/// In-flight counts per tenant, the request-to-tenant map, and the cap. +#[derive(Debug)] +pub(super) struct TenantInflight { + /// In-flight count of every active tenant. A tenant at zero has no entry. + by_tenant: HashMap, + /// The tenant of every admitted request not yet answered. + request_tenant: HashMap, + /// Requests the dispatcher holds across every core. + total_capacity: u32, + /// Most requests one tenant can hold in flight. Zero means no cap, which + /// holds only for a dispatcher with no capacity. + cap: u32, +} + +impl TenantInflight { + /// Accounting for a dispatcher that holds `total_capacity` requests. + pub(super) fn new(total_capacity: u32) -> Self { + Self { + by_tenant: HashMap::new(), + request_tenant: HashMap::new(), + total_capacity, + cap: total_capacity, + } + } + + /// The refusal for one more request of `tenant_id`, when it is at its cap. + pub(super) fn at_cap(&self, tenant_id: u64) -> Option { + if self.cap == 0 { + return None; + } + let inflight = self.inflight(tenant_id); + (inflight >= self.cap).then_some(TenantAtCap { + inflight, + cap: self.cap, + }) + } + + /// Count `request_id` in flight for `tenant_id`. A tenant that turns + /// active lowers the cap of every tenant. + pub(super) fn admit(&mut self, tenant_id: u64, request_id: u64) { + let count = self.by_tenant.entry(tenant_id).or_insert(0); + *count += 1; + let turned_active = *count == 1; + self.request_tenant.insert(request_id, tenant_id); + if turned_active { + self.recalculate_cap(); + } + } + + /// Release the slot `request_id` holds. Returns `true` when a slot was + /// freed. A tenant that turns idle raises the cap of every other tenant. + pub(super) fn release(&mut self, request_id: u64) -> bool { + let Some(tenant_id) = self.request_tenant.remove(&request_id) else { + return false; + }; + let Some(count) = self.by_tenant.get_mut(&tenant_id) else { + return false; + }; + *count = count.saturating_sub(1); + if *count == 0 { + self.by_tenant.remove(&tenant_id); + self.recalculate_cap(); + } + true + } + + /// Requests `tenant_id` holds in flight. + pub(super) fn inflight(&self, tenant_id: u64) -> u32 { + self.by_tenant.get(&tenant_id).copied().unwrap_or(0) + } + + /// The tenant of `request_id`, while it is in flight. + #[cfg(test)] + pub(super) fn tenant_of(&self, request_id: u64) -> Option { + self.request_tenant.get(&request_id).copied() + } + + /// The current per-tenant cap. + #[cfg(test)] + pub(super) fn cap(&self) -> u32 { + self.cap + } + + /// Set the cap to the total capacity split across the active tenants. + fn recalculate_cap(&mut self) { + if self.total_capacity == 0 { + return; + } + let active = u32::try_from(self.by_tenant.len()) + .unwrap_or(u32::MAX) + .max(1); + self.cap = (self.total_capacity / active).max(MIN_TENANT_CAP); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_single_active_tenant_holds_the_whole_capacity() { + let mut tenants = TenantInflight::new(8); + tenants.admit(1, 10); + assert_eq!(tenants.cap(), 8); + assert_eq!(tenants.at_cap(1), None); + } + + #[test] + fn a_second_active_tenant_halves_the_cap_and_its_exit_restores_it() { + let mut tenants = TenantInflight::new(8); + tenants.admit(1, 10); + tenants.admit(2, 20); + assert_eq!(tenants.cap(), 4); + + assert!(tenants.release(20)); + assert_eq!(tenants.cap(), 8); + assert_eq!(tenants.inflight(2), 0); + } + + #[test] + fn a_lower_cap_keeps_every_admitted_slot() { + let mut tenants = TenantInflight::new(8); + for request_id in 0..6 { + tenants.admit(1, request_id); + } + tenants.admit(2, 100); + assert_eq!(tenants.cap(), 4); + assert_eq!(tenants.inflight(1), 6, "admitted requests stay in flight"); + assert_eq!( + tenants.at_cap(1), + Some(TenantAtCap { + inflight: 6, + cap: 4 + }) + ); + for request_id in 0..6 { + assert!(tenants.release(request_id)); + } + assert_eq!(tenants.inflight(1), 0); + } + + #[test] + fn the_cap_never_falls_below_the_floor() { + let mut tenants = TenantInflight::new(4); + for tenant_id in 0..10 { + tenants.admit(tenant_id, tenant_id); + } + assert_eq!(tenants.cap(), MIN_TENANT_CAP); + } + + #[test] + fn releasing_an_unknown_request_frees_nothing() { + let mut tenants = TenantInflight::new(8); + assert!(!tenants.release(42)); + tenants.admit(1, 10); + assert!(tenants.release(10)); + assert!(!tenants.release(10)); + assert_eq!(tenants.tenant_of(10), None); + } + + #[test] + fn a_dispatcher_with_no_capacity_has_no_cap() { + let mut tenants = TenantInflight::new(0); + tenants.admit(1, 10); + assert_eq!(tenants.at_cap(1), None); + } +} diff --git a/nodedb/src/config/server/config.rs b/nodedb/src/config/server/config.rs index 2656ba073..7c24ecf69 100644 --- a/nodedb/src/config/server/config.rs +++ b/nodedb/src/config/server/config.rs @@ -434,6 +434,45 @@ mod tests { assert!(msg.contains("tuning.wal.write_buffer_size"), "{msg}"); } + #[test] + fn from_file_rejects_a_zero_calvin_backlog() { + let path = write_temp_config( + "nodedb-domain-calvin-backlog.toml", + "[tuning.calvin]\nmax_inflight_backlog = 0\n", + ); + let err = ServerConfig::from_file(&path).unwrap_err(); + std::fs::remove_file(&path).ok(); + let msg = err.to_string(); + assert!(msg.contains("tuning.calvin.max_inflight_backlog"), "{msg}"); + assert!(msg.contains("positive integer"), "{msg}"); + } + + #[test] + fn from_file_rejects_a_calvin_stall_warning_below_the_floor() { + let path = write_temp_config( + "nodedb-domain-calvin-stall.toml", + "[tuning.calvin]\nverdict_stall_warn_ms = 3\n", + ); + let err = ServerConfig::from_file(&path).unwrap_err(); + std::fs::remove_file(&path).ok(); + let msg = err.to_string(); + assert!(msg.contains("tuning.calvin.verdict_stall_warn_ms"), "{msg}"); + assert!(msg.contains("at least 4 milliseconds"), "{msg}"); + } + + #[test] + fn from_file_reads_the_calvin_section() { + let path = write_temp_config( + "nodedb-domain-calvin-ok.toml", + "[tuning.calvin]\nmax_inflight_backlog = 256\ncatch_up_window = 64\n", + ); + let cfg = ServerConfig::from_file(&path).expect("valid calvin section"); + std::fs::remove_file(&path).ok(); + assert_eq!(cfg.tuning.calvin.max_inflight_backlog, 256); + assert_eq!(cfg.tuning.calvin.catch_up_window, 64); + assert_eq!(cfg.tuning.calvin.channel_capacity, 512); + } + /// Every shipped default satisfies every bound the gate enforces. #[test] fn the_compiled_defaults_are_in_domain() { diff --git a/nodedb/src/config/server/domain.rs b/nodedb/src/config/server/domain.rs index bb2cd2950..bad851ab9 100644 --- a/nodedb/src/config/server/domain.rs +++ b/nodedb/src/config/server/domain.rs @@ -18,6 +18,10 @@ pub(super) const MIN_WAL_WRITE_BUFFER_BYTES: usize = 64 * 1024; /// shorter sweep costs more than the resolution it buys. pub(super) const MIN_SCOPE_EXPIRY_SECS: u64 = 10; +/// Smallest Calvin verdict stall warning interval. The scheduler's idle sweep +/// ticks at a quarter of it, and a zero-length tick panics the timer. +pub(super) const MIN_CALVIN_VERDICT_STALL_WARN_MS: u64 = 4; + /// Rejects an endpoint that carries no `http://` or `https://` host. pub(super) fn otlp_endpoint_has_host(raw: &str) -> bool { raw.strip_prefix("http://") @@ -103,6 +107,8 @@ pub(super) fn validate_domain(config: &ServerConfig) -> crate::Result<()> { )); } + validate_calvin(config)?; + if let Some(cluster) = config.cluster.as_ref() { positive_u64( u64::from(cluster.join_retry_max_attempts), @@ -129,3 +135,30 @@ pub(super) fn validate_domain(config: &ServerConfig) -> crate::Result<()> { Ok(()) } + +/// Checks `[tuning.calvin]`. A zero channel capacity panics the channel +/// constructor, and a zero bound stalls every scheduler. +fn validate_calvin(config: &ServerConfig) -> crate::Result<()> { + let calvin = &config.tuning.calvin; + positive_usize(calvin.channel_capacity, "tuning.calvin.channel_capacity")?; + positive_u64( + u64::from(calvin.txn_deadline_multiplier), + "tuning.calvin.txn_deadline_multiplier", + )?; + positive_u64( + calvin.dependent_read_passive_timeout_ms, + "tuning.calvin.dependent_read_passive_timeout_ms", + )?; + if calvin.verdict_stall_warn_ms < MIN_CALVIN_VERDICT_STALL_WARN_MS { + return Err(reject( + "tuning.calvin.verdict_stall_warn_ms", + calvin.verdict_stall_warn_ms, + "an interval of at least 4 milliseconds", + )); + } + positive_usize( + calvin.max_inflight_backlog, + "tuning.calvin.max_inflight_backlog", + )?; + positive_u64(calvin.catch_up_window, "tuning.calvin.catch_up_window") +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/applied_mirror.rs b/nodedb/src/control/cluster/calvin/scheduler/applied_mirror.rs index 82ed6f408..3ae5ed170 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/applied_mirror.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/applied_mirror.rs @@ -12,10 +12,14 @@ //! The mirror keeps the same shape as the gate: a fully-applied watermark and //! the applied positions above it. The tail is pruned as the watermark //! advances, so it stays as small as the gate's. +//! +//! [`AppliedMirrors`] also holds each running scheduler's caught-up state, +//! which startup reads before it opens the client gateway. use std::collections::{BTreeSet, HashMap}; use std::sync::{Arc, Mutex}; +use super::caught_up::CaughtUpRegistry; use super::recovery::NOT_YET_APPLIED_EPOCH; use crate::control::security::catalog::calvin_applied::StoredCalvinApplied; @@ -87,9 +91,16 @@ impl AppliedMirror { #[derive(Debug, Default)] pub struct AppliedMirrors { by_vshard: Mutex>>, + /// Whether each running scheduler reached its rebuild target. + caught_up: CaughtUpRegistry, } impl AppliedMirrors { + /// Caught-up state of every running scheduler on this node. + pub fn caught_up(&self) -> &CaughtUpRegistry { + &self.caught_up + } + /// Register the mirror of a scheduler starting for `vshard_id`. A /// restarted scheduler replaces its predecessor's mirror. pub fn register( diff --git a/nodedb/src/control/cluster/calvin/scheduler/caught_up.rs b/nodedb/src/control/cluster/calvin/scheduler/caught_up.rs new file mode 100644 index 000000000..1f9a24ba6 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/caught_up.rs @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Whether each running Calvin scheduler on this node reached its rebuild +//! target. +//! +//! A scheduler starts with a rebuild target: the highest epoch its recovery +//! found applied for the vShard. It is caught up once its fully-applied +//! watermark reaches that target. A scheduler with no target is caught up +//! from the start. Startup holds the client gateway until every running +//! scheduler is caught up. +//! +//! Each scheduler holds a [`CaughtUpHandle`] for its vShard. The handle +//! removes its entry when the scheduler is dropped, so the registry lists +//! only running schedulers. A restarted scheduler replaces its predecessor's +//! entry, and the predecessor's drop leaves the new entry in place. + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +type Entries = Arc>>>; + +/// Caught-up state of every running scheduler on this node, by vShard. +#[derive(Debug, Default)] +pub struct CaughtUpRegistry { + entries: Entries, +} + +impl CaughtUpRegistry { + /// Register the scheduler starting for `vshard_id`, not caught up yet. + pub fn register(&self, vshard_id: u32) -> CaughtUpHandle { + let flag = Arc::new(AtomicBool::new(false)); + self.entries + .lock() + .unwrap_or_else(|p| p.into_inner()) + .insert(vshard_id, Arc::clone(&flag)); + CaughtUpHandle { + vshard_id, + flag, + entries: Arc::clone(&self.entries), + } + } + + /// The vShards whose running scheduler is not caught up, in order. + pub fn lagging(&self) -> Vec { + self.entries + .lock() + .unwrap_or_else(|p| p.into_inner()) + .iter() + .filter(|(_, flag)| !flag.load(Ordering::Acquire)) + .map(|(vshard_id, _)| *vshard_id) + .collect() + } +} + +/// One scheduler's entry in the [`CaughtUpRegistry`]. +#[derive(Debug)] +pub struct CaughtUpHandle { + vshard_id: u32, + flag: Arc, + entries: Entries, +} + +impl CaughtUpHandle { + /// Whether this scheduler reported caught up. + pub fn is_caught_up(&self) -> bool { + self.flag.load(Ordering::Acquire) + } + + /// Report this scheduler caught up. The state never returns to lagging: + /// the fully-applied watermark only rises. + pub fn mark_caught_up(&self) { + self.flag.store(true, Ordering::Release); + } +} + +impl Drop for CaughtUpHandle { + fn drop(&mut self) { + let mut entries = self.entries.lock().unwrap_or_else(|p| p.into_inner()); + if entries + .get(&self.vshard_id) + .is_some_and(|flag| Arc::ptr_eq(flag, &self.flag)) + { + entries.remove(&self.vshard_id); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_registered_scheduler_lags_until_it_reports() { + let registry = CaughtUpRegistry::default(); + let first = registry.register(3); + let second = registry.register(1); + assert_eq!(registry.lagging(), vec![1, 3]); + + second.mark_caught_up(); + assert!(second.is_caught_up()); + assert_eq!(registry.lagging(), vec![3]); + + first.mark_caught_up(); + assert!(registry.lagging().is_empty()); + } + + #[test] + fn a_dropped_scheduler_leaves_the_registry() { + let registry = CaughtUpRegistry::default(); + let handle = registry.register(5); + assert_eq!(registry.lagging(), vec![5]); + drop(handle); + assert!(registry.lagging().is_empty()); + } + + #[test] + fn a_predecessor_drop_keeps_its_successor_entry() { + let registry = CaughtUpRegistry::default(); + let predecessor = registry.register(7); + let successor = registry.register(7); + drop(predecessor); + assert_eq!(registry.lagging(), vec![7], "the successor still lags"); + successor.mark_caught_up(); + assert!(registry.lagging().is_empty()); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/config.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/config.rs index a71084d62..32671d071 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/config.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/config.rs @@ -1,10 +1,14 @@ // SPDX-License-Identifier: BUSL-1.1 //! Configuration for the Calvin scheduler driver. +//! +//! The `[tuning.calvin]` section sets every knob except the epoch duration, +//! which comes from the sequencer config. use std::time::Duration; -use crate::bridge::dispatch::DATA_PLANE_QUEUE_CAPACITY; +use nodedb_cluster::calvin::SequencerConfig; +use nodedb_types::config::tuning::CalvinTuning; /// Tuning parameters for a [`super::core::Scheduler`] instance. #[derive(Debug, Clone)] @@ -13,11 +17,12 @@ pub struct SchedulerConfig { pub channel_capacity: usize, /// Transaction deadline multiplier (× epoch_duration_ms). pub txn_deadline_multiplier: u32, - /// Epoch duration in milliseconds (used to compute deadlines). + /// Epoch duration in milliseconds (used to compute deadlines). Equal to + /// the sequencer's epoch duration. pub epoch_duration_ms: u64, /// Timeout for passive participant responses in dependent-read txns. /// - /// Default: `epoch_duration_ms * 3` milliseconds. + /// Default: 60 milliseconds, three default epochs. /// /// `Instant::now()` is used for barrier timeouts (observability / off-WAL /// path only; never influences WAL bytes). @@ -29,7 +34,7 @@ pub struct SchedulerConfig { /// cross-shard commit. This is purely a liveness/observability knob: it /// controls how often the stall is surfaced, never whether the txn aborts. /// - /// Default: `epoch_duration_ms * 250` milliseconds. + /// Default: 5000 milliseconds, 250 default epochs. pub verdict_stall_warn_ms: u64, /// In-flight backlog at which the scheduler stops taking new sequenced /// input. The backlog counts pending, blocked, and dependent-barrier txns. @@ -38,35 +43,41 @@ pub struct SchedulerConfig { /// input. A backlog of blocked txns alone keeps intake open, because a /// reservation release they wait on arrives as input. /// - /// Default: [`DATA_PLANE_QUEUE_CAPACITY`]. Every dispatch of one vShard - /// goes to one Data Plane core, whose queue holds that many requests. A - /// larger backlog cannot hold a dispatch slot per txn at once. + /// Default: [`crate::bridge::dispatch::DATA_PLANE_QUEUE_CAPACITY`]. Every + /// dispatch of one vShard goes to one Data Plane core, whose queue holds + /// that many requests. A larger backlog cannot hold a dispatch slot per + /// txn at once. pub max_inflight_backlog: usize, /// Most sequencer log entries one catch-up drain reads and replays. /// - /// Default: `channel_capacity`. A drop happens only when the fan-out - /// channel is full, so one window covers about one channel of missed - /// entries. Windows run back to back while intake is open. + /// Default: 512, the default `channel_capacity`. A drop happens only when + /// the fan-out channel is full, so one window covers about one channel of + /// missed entries. Windows run back to back while intake is open. pub catch_up_window: u64, } impl Default for SchedulerConfig { fn default() -> Self { - let epoch_duration_ms = 20u64; - let channel_capacity = 512usize; - Self { - channel_capacity, - txn_deadline_multiplier: 3, - epoch_duration_ms, - dependent_read_passive_timeout_ms: epoch_duration_ms * 3, - verdict_stall_warn_ms: epoch_duration_ms * 250, - max_inflight_backlog: DATA_PLANE_QUEUE_CAPACITY, - catch_up_window: channel_capacity as u64, - } + Self::from_tuning(&CalvinTuning::default(), &SequencerConfig::default()) } } impl SchedulerConfig { + /// The scheduler config `tuning` sets, on the epoch duration of the + /// sequencer that feeds the scheduler. + pub fn from_tuning(tuning: &CalvinTuning, sequencer: &SequencerConfig) -> Self { + Self { + channel_capacity: tuning.channel_capacity, + txn_deadline_multiplier: tuning.txn_deadline_multiplier, + epoch_duration_ms: u64::try_from(sequencer.epoch_duration.as_millis()) + .unwrap_or(u64::MAX), + dependent_read_passive_timeout_ms: tuning.dependent_read_passive_timeout_ms, + verdict_stall_warn_ms: tuning.verdict_stall_warn_ms, + max_inflight_backlog: tuning.max_inflight_backlog, + catch_up_window: tuning.catch_up_window, + } + } + /// Passive timeout as a `Duration`. pub fn passive_timeout(&self) -> Duration { Duration::from_millis(self.dependent_read_passive_timeout_ms) @@ -77,3 +88,51 @@ impl SchedulerConfig { Duration::from_millis(self.verdict_stall_warn_ms) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::bridge::dispatch::DATA_PLANE_QUEUE_CAPACITY; + + #[test] + fn the_default_tuning_builds_the_shipped_scheduler_config() { + let config = SchedulerConfig::default(); + assert_eq!(config.channel_capacity, 512); + assert_eq!(config.txn_deadline_multiplier, 3); + assert_eq!(config.epoch_duration_ms, 20); + assert_eq!(config.dependent_read_passive_timeout_ms, 60); + assert_eq!(config.verdict_stall_warn_ms, 5_000); + assert_eq!(config.catch_up_window, 512); + } + + /// The tuning default restates the dispatcher queue capacity, which the + /// tuning crate cannot import. The two must not drift apart. + #[test] + fn the_default_backlog_is_one_dispatcher_queue() { + assert_eq!( + CalvinTuning::default().max_inflight_backlog, + DATA_PLANE_QUEUE_CAPACITY + ); + assert_eq!( + SchedulerConfig::default().max_inflight_backlog, + DATA_PLANE_QUEUE_CAPACITY + ); + } + + #[test] + fn tuning_overrides_reach_the_scheduler_config() { + let tuning = CalvinTuning { + max_inflight_backlog: 128, + verdict_stall_warn_ms: 9_000, + ..CalvinTuning::default() + }; + let sequencer = SequencerConfig { + epoch_duration: Duration::from_millis(50), + ..SequencerConfig::default() + }; + let config = SchedulerConfig::from_tuning(&tuning, &sequencer); + assert_eq!(config.max_inflight_backlog, 128); + assert_eq!(config.verdict_stall_warn(), Duration::from_millis(9_000)); + assert_eq!(config.epoch_duration_ms, 50); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_redo.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_redo.rs index 6ca3b632a..8a21b2fd6 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_redo.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_redo.rs @@ -345,7 +345,7 @@ mod tests { Some(HaltReason::ResolveFailed) ); assert_eq!( - scheduler.pending.get(&txn_id).and_then(|p| p.commit_state), + scheduler.pending.get(&txn_id).map(|p| p.commit_state), Some(CommitState::AwaitingRedoResolve), "no flush is dispatched" ); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/apply_tail.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/apply_tail.rs index 940ba2974..5445a93fa 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/apply_tail.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/apply_tail.rs @@ -14,6 +14,8 @@ use crate::control::cluster::calvin::scheduler::driver::core::halt::{ }; use crate::control::cluster::calvin::scheduler::driver::core::owed::SchedulerProposal; use crate::control::cluster::calvin::scheduler::driver::core::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::driver::core::write_version_record::VersionRecord; +use crate::control::cluster::calvin::scheduler::driver::types::CommitState; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; use crate::control::cluster::calvin::scheduler::metrics::infra_abort_reason; @@ -68,23 +70,22 @@ impl Scheduler { self.metrics.record_executor_error(); self.metrics .record_infra_abort(infra_abort_reason::IO_ERROR); - self.metrics.record_completed(); - self.on_txn_complete(txn_id); - return; } let completed = if committed { self.commit_apply_tail(txn_id, response, redo_lsn).await } else { // A dropped txn installed nothing, so its ack reports nothing. + // The coordinator's outcome waits for this ack, whatever the + // drop response said. self.propose_sequencer_entry( txn_id, SchedulerProposal::CompletionAck { result: Vec::new() }, ); true }; - // `false` means the commit tail halted the scheduler: the txn stays - // pending and unapplied. + // `false` means the txn stays pending and unapplied: the commit tail + // halted the scheduler, or its write-version record waits for capacity. if completed { self.metrics.record_completed(); self.on_txn_complete(txn_id); @@ -132,20 +133,18 @@ impl Scheduler { /// Deposit the applied result, durably mark the apply, record the apply's /// write versions, and propose the `CompletionAck`. /// - /// Shared by the flush-completion path and the direct-apply (dependent / - /// active) apply path. - /// - /// Returns `false` once a failed `CalvinApplied` WAL append halted the - /// scheduler: the position must not be marked applied without its marker, - /// so the caller leaves the txn pending. + /// Returns `true` when the caller completes the txn. Returns `false` when + /// the txn stays pending, with its position unapplied: + /// - a WAL append, its fsync, or the write-version record failed, and the + /// scheduler halted; + /// - the write-version record waits for capacity. The txn then moves to + /// [`CommitState::AwaitingVersionRecord`] and completes once the record is sent. /// /// `redo_lsn` is `Some(lsn)` when a `TransactionRedo` record was already /// WAL-appended for this commit's non-empty write set (`finish_redo_resolve`) /// — that record already IS the durable applied marker, so only write - /// versions are recorded at it. `None` (a drop, an empty-ops staged commit, - /// or the direct-apply dependent/active path, which carries no redo record) - /// falls back to appending a `CalvinApplied` marker here, exactly as before - /// this record existed. + /// versions are recorded at it. `None` (an empty-ops staged commit, which + /// carries no redo record) appends a `CalvinApplied` marker here. pub(in crate::control::cluster::calvin::scheduler::driver::core) async fn commit_apply_tail( &mut self, txn_id: TxnId, @@ -254,10 +253,7 @@ impl Scheduler { // SAME shard-local WAL-LSN space fast-path writes and read // watermarks use. Record the apply's per-key write versions at it; // no second (CalvinApplied) marker is written. - Some(lsn) => { - self.record_calvin_write_versions(txn_id, lsn); - Some(lsn) - } + Some(lsn) => Some((lsn, self.record_calvin_write_versions(txn_id, lsn))), None => match self .shared .wal @@ -272,10 +268,10 @@ impl Scheduler { // apply — the SAME shard-local WAL-LSN space fast-path writes and // read watermarks use. Record the apply's per-key write versions // at it once it exists; it does not exist yet at dispatch time. - Ok(applied_lsn) => { - self.record_calvin_write_versions(txn_id, applied_lsn); - Some(applied_lsn) - } + Ok(applied_lsn) => Some(( + applied_lsn, + self.record_calvin_write_versions(txn_id, applied_lsn), + )), Err(e) => { self.halt_apply( txn_id, @@ -287,13 +283,18 @@ impl Scheduler { } }, }; - let Some(lsn) = applied_lsn else { + let Some((lsn, version_record)) = applied_lsn else { // The apply cannot be acknowledged without a durable participant // LSN: CDC and write-version consumers will otherwise observe a // successful commit with no authoritative ordering point. The // scheduler halted above. return false; }; + if version_record == VersionRecord::Halted { + // Read-set validation reads the lost versions. The scheduler + // halted, and the txn stays pending and unapplied. + return false; + } // The record is whole at append: the flush reports no rows, so no // part follows it. A record split over the WAL record limit is // durable once its last continuation is. @@ -354,6 +355,14 @@ impl Scheduler { txn_id, SchedulerProposal::CompletionAck { result: ack_result }, ); + if version_record == VersionRecord::Parked { + // The txn keeps its locks until the record is sent, so no later + // txn validates a read before the versions land. + if let Some(pending) = self.pending.get_mut(&txn_id) { + pending.commit_state = CommitState::AwaitingVersionRecord; + } + return false; + } true } } @@ -545,7 +554,7 @@ mod tests { } /// A drop that returns an error under an abort verdict still completes - /// the txn: no replica writes an aborted txn. + /// the txn and owes its ack: no replica writes an aborted txn. #[tokio::test] async fn drop_error_response_under_abort_completes_txn() { let txn_id = TxnId::new(9, 2); @@ -564,6 +573,10 @@ mod tests { assert!(scheduler.applied.is_applied(9, 2)); assert!(!scheduler.pending.contains_key(&txn_id)); assert!(!scheduler.is_apply_halted()); + assert!(scheduler.owed.contains_key(&( + txn_id, + crate::control::cluster::calvin::scheduler::driver::core::owed::OwedKind::CompletionAck + ))); } fn retryable_refusal() -> Response { error_response(ErrorCode::RetryableRefusal { @@ -586,10 +599,10 @@ mod tests { let registry = nodedb_cluster::calvin::CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(7, registry); let mut pending = staged_pending(make_sequenced_txn(9, 2), txn_id); - pending.commit_state = Some(CommitState::AwaitingResolve { + pending.commit_state = CommitState::AwaitingResolve { committed: true, redo_lsn: None, - }); + }; pending.flush_scope.redo = vec![7, 7, 7]; pending.flush_scope.sends = 1; scheduler.pending.insert(txn_id, pending); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs index 2168b5c19..4b871743f 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs @@ -47,7 +47,7 @@ impl Scheduler { // Guard: only a still-parked txn resumes. Mirrors `handle_completion`'s // state-match so a duplicate push/probe/timeout is idempotent. if !matches!( - self.pending.get(&txn_id).and_then(|p| p.commit_state), + self.pending.get(&txn_id).map(|p| p.commit_state), Some(CommitState::AwaitingVerdict) ) { return; @@ -73,7 +73,7 @@ impl Scheduler { && let Some(pending) = self.pending.get_mut(&txn_id) && pending.flush_scope.resolve_at_turn { - pending.commit_state = Some(CommitState::AwaitingResolveTurn); + pending.commit_state = CommitState::AwaitingResolveTurn; pending.verdict_deadline = None; self.shared .calvin @@ -111,14 +111,14 @@ impl Scheduler { // this step's response, so a duplicate verdict push or probe is a // no-op under the guard above. if let Some(pending) = self.pending.get_mut(&txn_id) { - pending.commit_state = Some(if committed { + pending.commit_state = if committed { CommitState::AwaitingRedoResolve } else { CommitState::AwaitingResolve { committed: false, redo_lsn: None, } - }); + }; // No longer parked: clear the stall deadline. pending.verdict_deadline = None; } @@ -163,6 +163,10 @@ impl Scheduler { /// the transaction. The verdict is guaranteed to arrive eventually — a /// post-failover leader re-aggregates the replicated votes (seeded on every /// replica) into the same verdict — so waiting is always the safe action. + /// + /// The same stall tick re-proposes this vShard's vote while it is owed + /// (`retry_owed_sequencer_entries`). The warning names whether the vote is + /// still owed, so a stall that waits on another participant shows apart. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn check_awaiting_verdict_stalls( &mut self, ) { @@ -171,7 +175,7 @@ impl Scheduler { let stalled: Vec = self .pending .iter() - .filter(|(_, p)| matches!(p.commit_state, Some(CommitState::AwaitingVerdict))) + .filter(|(_, p)| p.commit_state == CommitState::AwaitingVerdict) .filter(|(_, p)| p.verdict_deadline.is_some_and(|d| now >= d)) .map(|(id, _)| *id) .collect(); @@ -187,10 +191,15 @@ impl Scheduler { // Verdict still unknown: keep waiting, hold locks, never abort. self.metrics.record_verdict_stall(); + let vote_owed = self.owed.contains_key(&( + txn_id, + crate::control::cluster::calvin::scheduler::driver::core::owed::OwedKind::Vote, + )); tracing::warn!( vshard_id = self.vshard_id, epoch = txn_id.epoch, position = txn_id.position, + vote_owed, "calvin: staged txn still awaiting the cross-shard verdict past its stall \ deadline; HOLDING locks and waiting (never aborting — a peer may have already \ flushed a commit). The verdict is guaranteed to arrive." @@ -242,7 +251,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, dir, mut data_side) = build_test_scheduler_with_data_side(7, registry); let mut pending = staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id); - pending.commit_state = Some(state); + pending.commit_state = state; scheduler.pending.insert(txn_id, pending); let shared = Arc::clone(&scheduler.shared); let fillers = fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); @@ -435,7 +444,7 @@ mod tests { scheduler .pending .get(&txn_id) - .and_then(|pending| pending.commit_state), + .map(|pending| pending.commit_state), Some(CommitState::AwaitingVerdict) )); assert!(data_side.request_rx.try_pop().is_err()); @@ -483,7 +492,7 @@ mod tests { scheduler .pending .get(&txn_id) - .and_then(|pending| pending.commit_state), + .map(|pending| pending.commit_state), Some(CommitState::AwaitingResolve { committed: false, redo_lsn: None @@ -542,7 +551,7 @@ mod tests { .pending .get(&txn_id) .expect("the txn stays pending"); - assert_eq!(pending.commit_state, Some(CommitState::AwaitingVerdict)); + assert_eq!(pending.commit_state, CommitState::AwaitingVerdict); assert_eq!(pending.verdict_deadline, None); assert_eq!( scheduler.apply_halt().map(|h| h.reason), diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs index 2a837fe53..f10a56bfe 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs @@ -84,10 +84,9 @@ impl Scheduler { self.propose_sequencer_entry(txn_id, SchedulerProposal::Vote { abort }); if vote == StagedVote::SerializationConflict { - // The staged slice's read-set was no longer current: observe it, the - // same node-global signal the direct-apply path records. A - // participant error never validated a read-set, so it must not count - // here. + // The staged slice's read-set was no longer current: observe it on + // the node-global counter. A participant error never validated a + // read-set, so it must not count here. self.shared .calvin .counters @@ -106,7 +105,7 @@ impl Scheduler { // `resume_on_verdict` halts on that COMMIT verdict. match self.pending.get_mut(&txn_id) { Some(pending) => { - pending.commit_state = Some(CommitState::AwaitingVerdict); + pending.commit_state = CommitState::AwaitingVerdict; // A superseded slice cannot flush here: its collection is gone. pending.stage_error = match vote { StagedVote::ParticipantError | StagedVote::PredictionDrift => { diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs index aee1145f2..99de7fa54 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs @@ -3,16 +3,15 @@ //! Executor-response routing for the Calvin scheduler. //! //! [`Scheduler::handle_completion`] is the `completion_rx` arm of the main -//! `select!` loop: it classifies each executor response (disconnect, OLLP -//! mismatch, staged commit-resolution state, or direct apply) and routes it to -//! the matching handler. Staged transactions thread through the commit-barrier -//! states in [`super::commit_resolve`]; a txn parked in +//! `select!` loop: it classifies each executor response (disconnect, or the +//! txn's commit-resolution state) and routes it to the matching handler. +//! Every txn threads through the commit-barrier states in +//! [`super::commit_resolve`]; a txn parked in //! [`CommitState::AwaitingVerdict`] has no outstanding bridge, so a completion //! for it is a no-op that keeps it parked. use super::super::types::CommitState; use super::halt::{HaltReason, HaltStep}; -use super::owed::SchedulerProposal; use super::scheduler::Scheduler; use crate::bridge::envelope::Response; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; @@ -28,6 +27,18 @@ impl Scheduler { request_id: RequestId, resp_opt: Option, ) { + let Some(commit_state) = self.pending.get(&txn_id).map(|p| p.commit_state) else { + // A response bridge exists only for a pending txn, and the txn + // leaves `pending` only after its last response. + tracing::error!( + vshard_id = self.vshard_id, + request_id = request_id.as_u64(), + epoch = txn_id.epoch, + position = txn_id.position, + "calvin: executor completion for a txn with no pending entry; ignoring" + ); + return; + }; let response = match resp_opt { Some(r) => r, None => { @@ -35,11 +46,10 @@ impl Scheduler { // so the request's outcome on this replica is unknown. Hold // the txn unapplied and halt. self.metrics.record_executor_error(); - let state = self.pending.get(&txn_id).and_then(|p| p.commit_state); self.halt_apply( txn_id, HaltReason::ResponseDisconnected, - HaltStep::awaited_by(state), + HaltStep::awaited_by(commit_state), format!( "executor response channel for request {} closed before a response", request_id.as_u64() @@ -56,99 +66,43 @@ impl Scheduler { .unwrap_or(0); self.metrics.record_executor_txn_duration_ms(elapsed_ms); - // A staged transaction resolves through its commit-barrier state, OLLP - // answer included. The flush installs a resolved redo record and runs - // no OLLP check, so an OLLP answer under a commit state is a failed - // step that halts and holds. It never settles as a retry. - let commit_state = self.pending.get(&txn_id).and_then(|p| p.commit_state); - - // OLLP mismatch: the active executor detected predicate drift and returned - // OllpRetryRequired without writing. The retry loop is COORDINATOR-owned - // (`run_dependent_with_retry`): the scheduler must NOT re-submit a stale - // prediction. Instead it (1) releases the aborted attempt's locks and - // (2) signals the coordinator's completion waiter via the registry so it - // can run a FRESH reconnaissance and resubmit. - if commit_state.is_none() - && response.status == crate::bridge::envelope::Status::Error - && response.error_code.as_deref() - == Some(&crate::bridge::envelope::ErrorCode::OllpRetryRequired) - { - // A mismatch is a normal OLLP retry signal, not a failure: the executor - // correctly detected predicate drift and declined to write. Count it as - // a received executor response, but NOT as an executor error or infra - // abort — those will inflate failure metrics on every routine retry. - self.metrics.record_completed(); - // (1) Release the aborted attempt's locks and clean up pending state. - // A retry that left the aborted attempt's locks held will leak them. - self.on_txn_complete(txn_id); - // OLLP mismatch broadcast is LEADER-ONLY. The optimistic-lock - // verification runs only on the data-group leader (the data plane - // skips it on followers — see the `ollp_is_group_leader` gate in the - // bulk-DML handlers), so by construction only a leader's executor can - // return `OllpRetryRequired`. This guard is defense-in-depth: a - // non-leader scheduler must never broadcast a mismatch — a lagging - // follower can otherwise poison an attempt the leader already - // completed, exhausting retries on a static dataset. A non-leader - // releases locks (done above) and returns; the leader owns the - // single mismatch signal that the completion registry observes. - if !self.is_group_leader() { - tracing::debug!( - vshard_id = self.vshard_id, - epoch = txn_id.epoch, - position = txn_id.position, - "calvin: OllpRetryRequired observed on non-leader; NOT broadcasting mismatch \ - (leader owns the verification decision)" - ); - return; - } - // (2) Broadcast the mismatch signal via the sequencer-group Raft so that - // the coordinator's CalvinCompletionRegistry fires on EVERY replica, - // including remote nodes. The SequencerStateMachine::apply() arm for - // OllpMismatch calls note_ollp_mismatch, waking the coordinator's - // retry-loop waiter wherever it is — mirrors how CompletionAck is - // delivered to remote coordinators. - self.propose_sequencer_entry(txn_id, SchedulerProposal::OllpMismatch); - return; - } - - // Staged static Calvin apply: a STAGED response carries the local commit - // vote; drive the vote-and-park and let the verdict resume the - // flush-or-drop. Dependent / active txns carry no `commit_state` and apply - // directly below. + // A txn resolves through its commit-barrier state, OLLP answer + // included. The flush installs a resolved redo record and runs no + // OLLP check, so an OLLP answer under a later state is a failed step + // that halts and holds. It never settles as a retry. match commit_state { - Some(CommitState::Staged) => { + CommitState::Staged => { // Stage failures remain participants in the barrier: // `resolve_staged_commit` turns any `Status::Error` into an // explicit false vote, parks locally, and waits for the // durable global verdict before issuing a drop. self.resolve_staged_commit(txn_id, &response); - return; } - Some(CommitState::AwaitingRedoResolve) => { + CommitState::AwaitingRedoResolve => { self.finish_redo_resolve(txn_id, response); - return; } - Some(CommitState::AwaitingResolve { + CommitState::AwaitingResolve { committed, redo_lsn, - }) => { + } => { self.finish_resolved_commit(txn_id, response, committed, redo_lsn) .await; - return; } - Some(CommitState::AwaitingFlushTurn { .. } | CommitState::AwaitingResolveTurn) => { + CommitState::AwaitingFlushTurn { .. } + | CommitState::AwaitingResolveTurn + | CommitState::AwaitingVersionRecord => { // A txn waiting for its flush or resolve turn has no request - // out. + // out. A write-version record drains its own response. tracing::warn!( vshard_id = self.vshard_id, request_id = request_id.as_u64(), epoch = txn_id.epoch, position = txn_id.position, - "calvin: executor completion for a txn waiting for its flush turn; ignoring" + ?commit_state, + "calvin: executor completion for a txn with no request out; ignoring" ); - return; } - Some(CommitState::AwaitingVerdict) => { + CommitState::AwaitingVerdict => { // A parked txn dispatched no executor request (it is waiting on // the cross-shard verdict, not the Data Plane), so no response // bridge is outstanding for it — a completion here indicates a @@ -163,49 +117,7 @@ impl Scheduler { "calvin: executor completion for a txn parked on the commit barrier; \ ignoring (no bridge is outstanding while AwaitingVerdict)" ); - return; } - None => {} - } - - if response.status != crate::bridge::envelope::Status::Ok { - // A failed direct apply must not leave the txn parked with its locks - // held: that wedges every txn queued behind those keys and freezes - // this vShard's epoch watermark, and no sweep re-drives the entry. - // Surface the infra abort and force completion. - tracing::error!( - vshard_id = self.vshard_id, - epoch = txn_id.epoch, - position = txn_id.position, - "calvin: executor response was not Ok; forcing infra-abort completion so locks \ - release and the epoch advances" - ); - self.metrics.record_executor_error(); - self.metrics.record_infra_abort( - crate::control::cluster::calvin::scheduler::metrics::infra_abort_reason::IO_ERROR, - ); - self.metrics.record_completed(); - self.on_txn_complete(txn_id); - return; - } - - // Observe whether the applying participant reported its slice of the - // transaction's reads as no longer current against the local write - // versions. Direct-apply (dependent/active) observation only: the - // staged path folds this into its commit vote instead. `None` means - // no read-set was checked. - if response.read_set_valid == Some(false) { - self.shared - .calvin - .counters - .read_set_validation_failures - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - } - // `false` means the commit tail halted the scheduler: the txn stays - // pending and unapplied. - if self.commit_apply_tail(txn_id, response, None).await { - self.metrics.record_completed(); - self.on_txn_complete(txn_id); } } } @@ -267,7 +179,7 @@ mod tests { let second = TxnId::new(6, 0); let (mut scheduler, _dir) = scheduler_with_pending(first, CommitState::Staged); let mut pending = staged_pending(make_sequenced_txn(6, 0), second); - pending.commit_state = Some(CommitState::AwaitingRedoResolve); + pending.commit_state = CommitState::AwaitingRedoResolve; scheduler.pending.insert(second, pending); scheduler diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs index a42623686..576b204ef 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs @@ -156,7 +156,7 @@ impl Scheduler { mut request, journal, } = parked; - if step != DispatchStep::WriteVersionRecord && !self.pending.contains_key(&txn_id) { + if !self.pending.contains_key(&txn_id) { tracing::error!( vshard_id = self.vshard_id, epoch = txn_id.epoch, @@ -168,7 +168,11 @@ impl Scheduler { } self.refresh_deferred_request(step, &mut request); match self.send_once(txn_id, step, request, journal) { - Attempt::Sent => {} + Attempt::Sent => { + if step == DispatchStep::WriteVersionRecord { + self.complete_after_version_record(txn_id); + } + } Attempt::Capacity(parked) => { self.deferred.push_front(*parked); break; @@ -188,11 +192,10 @@ impl Scheduler { /// Run the terminal handling of a dispatch the dispatcher refused for a /// reason other than capacity. /// - /// A stage, resolve, flush, or drop refusal halts the scheduler: the txn - /// keeps its `pending` entry and locks, and its position stays unapplied. - /// A refusal during shutdown holds the txn the same way. A write-version - /// record is dropped with a warning, because the commit does not depend on - /// it. + /// Every refusal halts the scheduler: the txn keeps its `pending` entry + /// and locks, and its position stays unapplied. A refusal during shutdown + /// holds the txn the same way. A lost write-version record halts too: + /// read-set validation reads those versions. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn fail_dispatch_step( &mut self, txn_id: TxnId, @@ -204,16 +207,7 @@ impl Scheduler { DispatchStep::Resolve => HaltStep::Resolve, DispatchStep::Flush => HaltStep::Flush, DispatchStep::Drop => HaltStep::Drop, - DispatchStep::WriteVersionRecord => { - tracing::warn!( - vshard_id = self.vshard_id, - epoch = txn_id.epoch, - position = txn_id.position, - %error, - "calvin: write-version record dispatch failed" - ); - return; - } + DispatchStep::WriteVersionRecord => HaltStep::WriteVersionRecord, }; self.halt_apply( txn_id, diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs index 27058bed9..74329ac00 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs @@ -5,8 +5,6 @@ use std::time::Instant; -use tracing::error; - use nodedb_cluster::calvin::types::SequencedTxn; use nodedb_physical::physical_plan::PhysicalPlan; use nodedb_physical::physical_plan::meta::MetaOp; @@ -38,14 +36,7 @@ impl Scheduler { let plans = match super::super::super::helpers::decode_plans(&txn.tx_class.plans) { Ok(p) => p, Err(e) => { - error!( - vshard_id = self.vshard_id, - epoch, - position, - error = %e, - "calvin scheduler: active plan decode failed; releasing locks" - ); - self.on_unpending_txn_complete(txn_id, lock_owner); + self.reject_plan(txn, txn_id, lock_owner, &e); return; } }; @@ -62,9 +53,9 @@ impl Scheduler { // A dependent-read active txn dispatched here always carries a // local write slice (the OLLP orchestrator only routes the write // participant through this path). An empty local slice is a - // routing bug, not a read-only participant — surface it as a - // terminal routing failure rather than dispatching an - // active task with nothing to apply. + // routing bug, not a read-only participant, so it rejects the + // plans rather than dispatching an active task with nothing + // to apply. let e = crate::Error::Internal { detail: format!( "calvin active txn {epoch}/{position} homes no local write plans \ @@ -72,27 +63,11 @@ impl Scheduler { self.vshard_id ), }; - error!( - vshard_id = self.vshard_id, - epoch, - position, - error = %e, - "calvin scheduler: active txn homes no local writes; releasing locks" - ); - self.propose_routing_failure(txn_id, &e); - self.on_unpending_txn_complete(txn_id, lock_owner); + self.reject_plan(txn, txn_id, lock_owner, &e); return; } Err(e) => { - error!( - vshard_id = self.vshard_id, - epoch, - position, - error = %e, - "calvin scheduler: active txn routing failed; releasing locks" - ); - self.propose_routing_failure(txn_id, &e); - self.on_unpending_txn_complete(txn_id, lock_owner); + self.reject_plan(txn, txn_id, lock_owner, &e); return; } }; @@ -138,7 +113,7 @@ impl Scheduler { // resolve → redo → flush as the static path, for // WAL-only-restart durability. `resolve_staged_commit` reads the // `read_set_valid: None` the active handler returns as "commit". - commit_state: Some(super::super::super::types::CommitState::Staged), + commit_state: super::super::super::types::CommitState::Staged, // Set only once the txn parks in `AwaitingVerdict`. verdict_deadline: None, stage_error: None, diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs index 8ad904fb2..d23c6dc0a 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs @@ -150,14 +150,14 @@ impl Scheduler { } /// Whether a txn in `state` can have a write in the Data Plane's hands: a -/// committed flush, or a direct apply. -fn writes_in_flight(state: Option) -> bool { +/// committed flush. +fn writes_in_flight(state: CommitState) -> bool { matches!( state, - None | Some(CommitState::AwaitingResolve { + CommitState::AwaitingResolve { committed: true, .. - }) + } ) } @@ -243,7 +243,7 @@ mod tests { } scheduler.resolve_staged_commit(txn_id, &staged_response(Status::Ok, Some(true))); let pending = scheduler.pending.get(&txn_id).expect("still parked"); - assert_eq!(pending.commit_state, Some(CommitState::AwaitingVerdict)); + assert_eq!(pending.commit_state, CommitState::AwaitingVerdict); assert!(pending.stage_error.is_some(), "a COMMIT verdict halts it"); } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/mod.rs index 303ce0094..f71c8d1c1 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/mod.rs @@ -7,3 +7,4 @@ mod bind_identities; mod incarnation; mod primary_write; mod static_dispatch; +mod unstaged; diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/static_dispatch.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/static_dispatch.rs index 22b531f89..f3983108e 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/static_dispatch.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/static_dispatch.rs @@ -5,14 +5,11 @@ use std::time::Instant; -use tracing::error; - use nodedb_cluster::calvin::types::SequencedTxn; use nodedb_physical::physical_plan::PhysicalPlan; use nodedb_physical::physical_plan::meta::MetaOp; use super::super::deferred::{DispatchOutcome, DispatchStep}; -use super::super::owed::SchedulerProposal; use super::super::routing::PlanRouting; use super::super::scheduler::Scheduler; use super::primary_write::{ @@ -53,27 +50,6 @@ impl Scheduler { mr.vshard_role_is_leader(self.vshard_id) } - /// Broadcast a terminal, NON-retryable routing-failure signal via the - /// sequencer-group Raft so every replica's `CalvinCompletionRegistry` - /// fires `note_routing_failed`, waking the coordinator's completion - /// waiter immediately with the reason instead of leaving it to burn the - /// full deadline and report a generic timeout. Mirrors the OllpMismatch - /// broadcast in `handle_executor_response`. Shared by `dispatch_txn` and - /// `dispatch_active_txn`, and by a multi-part transaction whose part - /// fails to decode. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn propose_routing_failure( - &mut self, - txn_id: TxnId, - err: &crate::Error, - ) { - self.propose_sequencer_entry( - txn_id, - SchedulerProposal::RoutingFailed { - detail: err.to_string(), - }, - ); - } - /// Filter a transaction's write plans down to the slice that homes to this /// scheduler's vShard. /// @@ -147,14 +123,7 @@ impl Scheduler { let plans = match super::super::super::helpers::decode_plans(&txn.tx_class.plans) { Ok(p) => p, Err(e) => { - error!( - vshard_id = self.vshard_id, - epoch, - position, - error = %e, - "calvin scheduler: plan decode failed; releasing locks and skipping txn" - ); - self.on_unpending_txn_complete(txn_id, lock_owner); + self.reject_plan(txn, txn_id, lock_owner, &e); return; } }; @@ -176,15 +145,7 @@ impl Scheduler { match self.local_calvin_plans(plans, txn.tx_class.database_id, epoch, position) { Ok(p) => p, Err(e) => { - error!( - vshard_id = self.vshard_id, - epoch, - position, - error = %e, - "calvin scheduler: static txn routing failed; releasing locks" - ); - self.propose_routing_failure(txn_id, &e); - self.on_unpending_txn_complete(txn_id, lock_owner); + self.reject_plan(txn, txn_id, lock_owner, &e); return; } }; @@ -196,7 +157,7 @@ impl Scheduler { // participant — writes home elsewhere, but a read homes HERE, so it must // still validate its slice of the read-set and cast a real commit/abort // vote — or a routing bug (neither writes nor reads home here). Only the - // latter is an error; the former stages a validate-only task below. + // latter rejects the plans; the former stages a validate-only task below. if local.is_empty() && !super::super::routing::homes_versioned_read( &txn.tx_class.versioned_reads, @@ -211,15 +172,7 @@ impl Scheduler { self.vshard_id ), }; - error!( - vshard_id = self.vshard_id, - epoch, - position, - error = %e, - "calvin scheduler: static txn homes no local work; releasing locks" - ); - self.propose_routing_failure(txn_id, &e); - self.on_unpending_txn_complete(txn_id, lock_owner); + self.reject_plan(txn, txn_id, lock_owner, &e); return; } @@ -349,7 +302,7 @@ impl Scheduler { // This dispatch STAGES the txn (validate + buffer, no apply); // its response carries the local commit vote that drives the // subsequent flush-or-drop. - commit_state: Some(super::super::super::types::CommitState::Staged), + commit_state: super::super::super::types::CommitState::Staged, // Set only once the txn parks in `AwaitingVerdict`. verdict_deadline: None, stage_error: None, diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs new file mode 100644 index 000000000..e7c0bea10 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A transaction this vShard cannot stage: its plans are rejected, or its +//! dependent reads never arrived. +//! +//! The txn still takes the commit barrier. It enters `pending`, holds its +//! locks, and owes an abort vote. At the abort verdict it drops like any +//! staged txn: it proposes its `CompletionAck`, releases its locks, and only +//! then marks its position applied. + +use std::time::Instant; + +use tracing::error; + +use nodedb_cluster::calvin::AbortReason; +use nodedb_cluster::calvin::types::SequencedTxn; + +use super::super::owed::SchedulerProposal; +use super::super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::driver::types::{ + CommitState, FlushScope, PendingTxn, +}; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + +impl Scheduler { + /// Park `txn` with an abort vote for plans this vShard rejects. + /// + /// Plan decode, part assembly, and plan routing read only replicated + /// input. So every replica rejects the same plans, and every replica + /// votes `PlanRejected`. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn reject_plan( + &mut self, + txn: SequencedTxn, + txn_id: TxnId, + lock_owner: TxnId, + error: &crate::Error, + ) { + error!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + error = %error, + "calvin scheduler: plans rejected; voting abort" + ); + self.park_unstaged( + txn, + txn_id, + lock_owner, + AbortReason::PlanRejected, + error.to_string(), + ); + } + + /// Park `txn`, which never staged on this replica, on the commit barrier + /// with an abort vote for `reason`. + /// + /// `stage_error` stays on the pending entry. A COMMIT verdict for the + /// txn then halts the scheduler, because this replica holds nothing to + /// flush. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn park_unstaged( + &mut self, + txn: SequencedTxn, + txn_id: TxnId, + lock_owner: TxnId, + reason: AbortReason, + stage_error: String, + ) { + // no-determinism: dispatch time and stall deadline are observability only; the replicated verdict decides. + let now = Instant::now(); + self.pending.insert( + txn_id, + PendingTxn { + txn, + lock_owner, + dispatch_time: now, + has_primary_write: false, + has_returning: false, + change_sets: Vec::new(), + commit_state: CommitState::AwaitingVerdict, + verdict_deadline: Some(now + self.config.verdict_stall_warn()), + stage_error: Some(stage_error), + redo_records: None, + flush_scope: FlushScope::default(), + superseded: false, + gates: Vec::new(), + ungated: false, + install_permit: None, + }, + ); + self.propose_sequencer_entry( + txn_id, + SchedulerProposal::Vote { + abort: Some(reason), + }, + ); + // The verdict can be stored already: on replay, or when a peer's + // vote completed the tally first. + if let Some(verdict) = self.registry.verdict(nodedb_cluster::calvin::TxnId::new( + txn_id.epoch, + txn_id.position, + )) { + self.resume_on_verdict(txn_id, verdict); + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use nodedb_cluster::calvin::types::SchedulerInput; + use nodedb_cluster::calvin::{ + AttemptOutcome, CalvinCompletionRegistry, ParticipantVote, SequencerEntry, + }; + + use super::*; + use crate::bridge::dispatch::CoreChannelDataSide; + use crate::bridge::envelope::Status; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ + CapturingProposer, elect_data_group_leader, + }; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler_with_data_side, make_sequenced_txn, staged_pending, staged_response, + }; + + const REJECTING: u32 = 7; + const PEER: u32 = 8; + + /// A scheduler on `vshard` that leads its data group and captures its + /// proposals. + fn leading_scheduler( + vshard: u32, + registry: &Arc, + ) -> ( + Scheduler, + tempfile::TempDir, + CoreChannelDataSide, + Arc, + ) { + let (mut scheduler, dir, data_side) = + build_test_scheduler_with_data_side(vshard, Arc::clone(registry)); + let proposer = CapturingProposer::accepting(); + scheduler.sequencer_proposer = proposer.clone(); + elect_data_group_leader(&scheduler); + (scheduler, dir, data_side, proposer) + } + + /// Apply every vote `proposer` accepted to `registry`, as the sequencer + /// log applies them on every replica. + fn apply_votes(registry: &CalvinCompletionRegistry, proposer: &CapturingProposer) { + for entry in proposer.accepted() { + if let SequencerEntry::Vote { + epoch, + position, + vshard, + } = entry + { + registry.note_vote( + nodedb_cluster::calvin::TxnId::new(epoch, position), + vshard, + ParticipantVote::Commit, + ); + } else if let SequencerEntry::AbortVote { + epoch, + position, + vshard, + reason, + } = entry + { + registry.note_vote( + nodedb_cluster::calvin::TxnId::new(epoch, position), + vshard, + ParticipantVote::Abort(reason), + ); + } + } + } + + /// A txn whose plan bytes do not decode. + fn undecodable_txn(epoch: u64, position: u32) -> SequencedTxn { + let mut txn = make_sequenced_txn(epoch, position); + txn.tx_class.plans = vec![0xc1]; + txn + } + + /// A rejected plan votes `PlanRejected`. The tally completes with the + /// peer's commit vote, the verdict aborts, and both participants drop, + /// ack, release their locks, and mark the position applied. + #[tokio::test] + async fn a_rejected_plan_aborts_every_participant_through_the_verdict() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut rejecting, _dir_r, _data_r, proposer_r) = leading_scheduler(REJECTING, ®istry); + let (mut peer, _dir_p, _data_p, proposer_p) = leading_scheduler(PEER, ®istry); + let txn_id = TxnId::new(30, 0); + let cluster_txn = nodedb_cluster::calvin::TxnId::new(30, 0); + registry.seed_expected(cluster_txn, 2); + let outcome = registry.register_completion(cluster_txn, 2); + + rejecting.process_scheduler_input(SchedulerInput::Txn(Box::new(undecodable_txn(30, 0)))); + // A later txn on the same key waits behind the rejected txn's locks. + rejecting.process_scheduler_input(SchedulerInput::Txn(Box::new(undecodable_txn(31, 0)))); + assert!(rejecting.blocked.contains_key(&TxnId::new(31, 0))); + assert_eq!( + rejecting.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingVerdict) + ); + assert!(!rejecting.applied.is_applied(30, 0)); + + peer.pending + .insert(txn_id, staged_pending(make_sequenced_txn(30, 0), txn_id)); + peer.resolve_staged_commit(txn_id, &staged_response(Status::Ok, Some(true))); + assert_eq!( + peer.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingVerdict) + ); + + apply_votes(®istry, &proposer_r); + apply_votes(®istry, &proposer_p); + assert!(proposer_r.accepted().contains(&SequencerEntry::AbortVote { + epoch: 30, + position: 0, + vshard: REJECTING, + reason: AbortReason::PlanRejected, + })); + let verdicts = registry.drain_unproposed_verdicts(); + assert_eq!( + verdicts, + vec![( + cluster_txn, + nodedb_cluster::calvin::VerdictOutcome::Abort(AbortReason::PlanRejected) + )] + ); + registry.note_verdict(cluster_txn, verdicts[0].1); + + for (scheduler, vshard) in [(&mut rejecting, REJECTING), (&mut peer, PEER)] { + scheduler.resume_on_verdict(txn_id, false); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingResolve { + committed: false, + redo_lsn: None + }), + "vShard {vshard} leaves AwaitingVerdict for its drop" + ); + assert!(!scheduler.applied.is_applied(30, 0)); + scheduler + .finish_resolved_commit(txn_id, staged_response(Status::Ok, None), false, None) + .await; + assert!(!scheduler.pending.contains_key(&txn_id)); + assert!(scheduler.applied.is_applied(30, 0)); + registry.note_completion_ack(cluster_txn, vshard); + } + assert!( + !rejecting.blocked.contains_key(&TxnId::new(31, 0)), + "the released locks promote the waiting txn" + ); + assert_eq!( + outcome.await.expect("outcome fires"), + AttemptOutcome::Aborted { + reason: AbortReason::PlanRejected + } + ); + } + + /// A verdict stored before the rejection resumes the txn at once. + #[tokio::test] + async fn a_rejection_after_the_verdict_drops_at_once() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, _data, _proposer) = leading_scheduler(REJECTING, ®istry); + let txn_id = TxnId::new(32, 0); + registry.note_verdict( + nodedb_cluster::calvin::TxnId::new(32, 0), + nodedb_cluster::calvin::VerdictOutcome::Abort(AbortReason::PlanRejected), + ); + + scheduler.dispatch_txn(undecodable_txn(32, 0), txn_id, txn_id); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingResolve { + committed: false, + redo_lsn: None + }) + ); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/flush_turn.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/flush_turn.rs index 82ed6d1ca..dffa37acd 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/flush_turn.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/flush_turn.rs @@ -32,7 +32,7 @@ impl Scheduler { redo_lsn: Option, ) { if let Some(pending) = self.pending.get_mut(&txn_id) { - pending.commit_state = Some(CommitState::AwaitingFlushTurn { redo_lsn }); + pending.commit_state = CommitState::AwaitingFlushTurn { redo_lsn }; } self.pump_flush_turn(); } @@ -46,7 +46,7 @@ impl Scheduler { let resolve_turn = self .pending .get(&lowest) - .is_some_and(|pending| pending.commit_state == Some(CommitState::AwaitingResolveTurn)); + .is_some_and(|pending| pending.commit_state == CommitState::AwaitingResolveTurn); if resolve_turn { self.dispatch_resolve_turn(lowest); return; @@ -55,7 +55,7 @@ impl Scheduler { self.pending .get(&lowest) .and_then(|pending| match pending.commit_state { - Some(CommitState::AwaitingFlushTurn { redo_lsn }) => Some(redo_lsn), + CommitState::AwaitingFlushTurn { redo_lsn } => Some(redo_lsn), _ => None, }) else { @@ -79,10 +79,10 @@ impl Scheduler { return; } if let Some(pending) = self.pending.get_mut(&lowest) { - pending.commit_state = Some(CommitState::AwaitingResolve { + pending.commit_state = CommitState::AwaitingResolve { committed: true, redo_lsn, - }); + }; } } @@ -94,7 +94,7 @@ impl Scheduler { return; } if let Some(pending) = self.pending.get_mut(&txn_id) { - pending.commit_state = Some(CommitState::AwaitingRedoResolve); + pending.commit_state = CommitState::AwaitingRedoResolve; } } @@ -143,7 +143,7 @@ mod tests { scheduler.queue_flush(high, None); assert_eq!( - scheduler.pending.get(&high).and_then(|p| p.commit_state), + scheduler.pending.get(&high).map(|p| p.commit_state), Some(CommitState::AwaitingFlushTurn { redo_lsn: None }), "the lower txn is unfinished, so the higher flush waits" ); @@ -151,7 +151,7 @@ mod tests { scheduler.pending.remove(&low); scheduler.pump_flush_turn(); assert_eq!( - scheduler.pending.get(&high).and_then(|p| p.commit_state), + scheduler.pending.get(&high).map(|p| p.commit_state), Some(CommitState::AwaitingResolve { committed: true, redo_lsn: None, @@ -177,15 +177,12 @@ mod tests { } if let Some(pending) = scheduler.pending.get_mut(&truncate) { pending.flush_scope.resolve_at_turn = true; - pending.commit_state = Some(CommitState::AwaitingResolveTurn); + pending.commit_state = CommitState::AwaitingResolveTurn; } scheduler.pump_flush_turn(); assert_eq!( - scheduler - .pending - .get(&truncate) - .and_then(|p| p.commit_state), + scheduler.pending.get(&truncate).map(|p| p.commit_state), Some(CommitState::AwaitingResolveTurn), "the lower txn is unfinished, so the resolve waits" ); @@ -193,10 +190,7 @@ mod tests { scheduler.pending.remove(&low); scheduler.pump_flush_turn(); assert_eq!( - scheduler - .pending - .get(&truncate) - .and_then(|p| p.commit_state), + scheduler.pending.get(&truncate).map(|p| p.commit_state), Some(CommitState::AwaitingRedoResolve), "with the lower txn finished, the resolve dispatches" ); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs index 33b24843c..9be76f2ec 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs @@ -62,6 +62,9 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) enum HaltReason /// The metadata group left this node before its apply reached a held /// txn's floor. MetadataGroupGone, + /// A committed txn's write-version record cannot be built. Read-set + /// validation reads these versions, so a lost record reports false-valid reads. + WriteVersionRecordFailed, } impl HaltReason { @@ -77,6 +80,7 @@ impl HaltReason { Self::IdentityBindFailed => apply_halt_reason::IDENTITY_BIND_FAILED, Self::WalAppendFailed => apply_halt_reason::WAL_APPEND_FAILED, Self::MetadataGroupGone => apply_halt_reason::METADATA_GROUP_GONE, + Self::WriteVersionRecordFailed => apply_halt_reason::WRITE_VERSION_RECORD_FAILED, } } @@ -99,8 +103,8 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) enum HaltStep { Flush, /// The drop request of an aborted txn. Drop, - /// The direct apply of a txn with no commit state. - Apply, + /// The write-version record of a committed txn. + WriteVersionRecord, /// The `TransactionRedo` WAL append. RedoAppend, /// The `CalvinApplied` WAL append. @@ -121,7 +125,7 @@ impl HaltStep { Self::Resolve => "resolve", Self::Flush => "flush", Self::Drop => "drop", - Self::Apply => "apply", + Self::WriteVersionRecord => "write_version_record", Self::RedoAppend => "redo_append", Self::AppliedMarker => "applied_marker", Self::IdentityBind => "identity_bind", @@ -131,21 +135,19 @@ impl HaltStep { /// The request a txn in `state` awaits a response for. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn awaited_by( - state: Option, + state: CommitState, ) -> Self { match state { - Some(CommitState::Staged | CommitState::AwaitingVerdict) => Self::Stage, - Some(CommitState::AwaitingRedoResolve | CommitState::AwaitingResolveTurn) => { - Self::Resolve - } - Some(CommitState::AwaitingFlushTurn { .. }) => Self::Flush, - Some(CommitState::AwaitingResolve { + CommitState::Staged | CommitState::AwaitingVerdict => Self::Stage, + CommitState::AwaitingRedoResolve | CommitState::AwaitingResolveTurn => Self::Resolve, + CommitState::AwaitingFlushTurn { .. } => Self::Flush, + CommitState::AwaitingResolve { committed: true, .. - }) => Self::Flush, - Some(CommitState::AwaitingResolve { + } => Self::Flush, + CommitState::AwaitingResolve { committed: false, .. - }) => Self::Drop, - None => Self::Apply, + } => Self::Drop, + CommitState::AwaitingVersionRecord => Self::WriteVersionRecord, } } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs index d4895393b..3da3caa6d 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs @@ -29,6 +29,7 @@ pub mod intake; mod metadata_hold; pub mod owed; pub mod parts; +mod parts_error; pub mod parts_lane; pub mod process; pub mod propose; diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/entry.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/entry.rs index 51f485d9a..6f3f43733 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/entry.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/entry.rs @@ -25,10 +25,6 @@ pub enum SchedulerProposal { /// the coordinator reads: the timeseries install counts of the apply. /// Empty when there are none. CompletionAck { result: Vec }, - /// The active executor saw its OLLP prediction drift. - OllpMismatch, - /// The txn's local plan routing failed for good. - RoutingFailed { detail: String }, } impl SchedulerProposal { @@ -37,8 +33,6 @@ impl SchedulerProposal { match self { Self::Vote { .. } => OwedKind::Vote, Self::CompletionAck { .. } => OwedKind::CompletionAck, - Self::OllpMismatch => OwedKind::OllpMismatch, - Self::RoutingFailed { .. } => OwedKind::RoutingFailed, } } @@ -67,12 +61,6 @@ impl SchedulerProposal { result: result.clone(), from_node: node_id, }, - Self::OllpMismatch => SequencerEntry::OllpMismatch { epoch, position }, - Self::RoutingFailed { detail } => SequencerEntry::TxnRoutingFailed { - epoch, - position, - detail: detail.clone(), - }, } } } @@ -82,8 +70,6 @@ impl SchedulerProposal { pub enum OwedKind { Vote, CompletionAck, - OllpMismatch, - RoutingFailed, } impl OwedKind { @@ -97,24 +83,17 @@ impl OwedKind { match self { Self::Vote => sequencer_propose_kind::VOTE, Self::CompletionAck => sequencer_propose_kind::COMPLETION_ACK, - Self::OllpMismatch => sequencer_propose_kind::OLLP_MISMATCH, - Self::RoutingFailed => sequencer_propose_kind::ROUTING_FAILED, } } - /// Whether only the vShard's data-group leader proposes this kind. - pub fn leader_only(self) -> bool { - matches!(self, Self::Vote | Self::CompletionAck) - } - /// Whether this node applied the entry of this kind. /// /// `progress` is the registry's view of the txn for the scheduler's /// vShard. `entry_seen` is whether the registry held an entry for the - /// txn at an earlier check. The registry removes an entry only once the - /// txn's outcome fired, and a txn with a fired outcome needs no entry of - /// any kind. A missing entry that was never seen means this node has not - /// seeded the txn, so the entry is still owed. + /// txn at an earlier check. The registry evicts only a terminal entry: + /// its verdict is stored and every participant acked, so the txn needs + /// no entry of any kind. A missing entry that was never seen means this + /// node has not seeded the txn, so the entry is still owed. /// /// A stored verdict also settles a vote: the verdict forms only once /// every participant's vote is in the tally. @@ -125,8 +104,6 @@ impl OwedKind { match self { Self::Vote => p.voted || p.has_verdict, Self::CompletionAck => p.acked, - Self::OllpMismatch => p.mismatched, - Self::RoutingFailed => p.routing_failed, } } } @@ -146,8 +123,8 @@ pub struct OwedEntry { /// Owed entries keyed by txn and kind, in deterministic order. /// /// Bounded: it holds at most one entry per (txn, kind), and a txn owes at -/// most two kinds (a vote and a completion ack, or a single terminal -/// signal). An entry leaves once this node applies it. Entries pile up only +/// most two kinds, a vote and a completion ack. An entry leaves once this +/// node applies it. Entries pile up only /// while no sequencer leader is reachable, and then the sequencer admits no /// new txns either. pub type OwedEntries = BTreeMap<(TxnId, OwedKind), OwedEntry>; @@ -161,17 +138,10 @@ mod tests { voted: false, acked: false, has_verdict: false, - mismatched: false, - routing_failed: false, } } - const ALL_KINDS: [OwedKind; 4] = [ - OwedKind::Vote, - OwedKind::CompletionAck, - OwedKind::OllpMismatch, - OwedKind::RoutingFailed, - ]; + const ALL_KINDS: [OwedKind; 2] = [OwedKind::Vote, OwedKind::CompletionAck]; #[test] fn nothing_is_applied_while_the_registry_shows_no_effect() { @@ -190,20 +160,10 @@ mod tests { acked: true, ..progress() }; - let mismatched = ParticipantProgress { - mismatched: true, - ..progress() - }; - let routing_failed = ParticipantProgress { - routing_failed: true, - ..progress() - }; assert!(OwedKind::Vote.is_applied(Some(voted), false)); assert!(!OwedKind::CompletionAck.is_applied(Some(voted), false)); assert!(OwedKind::CompletionAck.is_applied(Some(acked), false)); assert!(!OwedKind::Vote.is_applied(Some(acked), false)); - assert!(OwedKind::OllpMismatch.is_applied(Some(mismatched), false)); - assert!(OwedKind::RoutingFailed.is_applied(Some(routing_failed), false)); } #[test] @@ -247,5 +207,15 @@ mod tests { reason: AbortReason::SerializationConflict } )); + assert!(matches!( + SchedulerProposal::Vote { + abort: Some(AbortReason::PlanRejected) + } + .entry(txn, 9, 1), + SequencerEntry::AbortVote { + reason: AbortReason::PlanRejected, + .. + } + )); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/retry.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/retry.rs index 7ddf13a87..a2df9532d 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/retry.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/retry.rs @@ -5,9 +5,7 @@ //! Re-proposing is safe because every entry kind applies idempotently in the //! completion registry. A vote is stored per vShard, so a repeat overwrites //! it with the same value, and the verdict it completes is emitted once. An -//! ack is a set insert per vShard, and the completion fires once. The -//! mismatch and routing-failure signals set a flag, and each fires its -//! waiter once. +//! ack is a set insert per vShard, and the completion fires once. use tracing::{debug, error}; @@ -24,6 +22,13 @@ impl Scheduler { /// The entry stays owed until this node's completion registry shows it /// applied. [`Self::retry_owed_sequencer_entries`] proposes it again on /// the stall tick until then. + /// + /// A vote and a `CompletionAck` answer for the vShard on every node: the + /// first one the sequencer log holds counts. So only the vShard's + /// data-group leader proposes them. A node that left the group leads none + /// of it, so an entry from its stale state never enters the log. Every + /// other replica keeps the entry owed and proposes it if it comes to lead + /// before one applies. Which entry counts then depends on the log alone. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn propose_sequencer_entry( &mut self, txn_id: TxnId, @@ -46,7 +51,7 @@ impl Scheduler { } }; let entry_seen = self.participant_progress(txn_id).is_some(); - let in_flight = self.may_propose(kind) + let in_flight = self.is_group_leader() && propose_owed( self.sequencer_proposer.as_ref(), self.vshard_id, @@ -81,11 +86,12 @@ impl Scheduler { !kind.is_applied(progress, owed.entry_seen) }); + if !leads { + // Every entry stays owed: this replica proposes it if it comes + // to lead. + return; + } for ((txn_id, kind), owed) in self.owed.iter_mut() { - if kind.leader_only() && !leads { - // Stays owed: this replica proposes it if it comes to lead. - continue; - } if owed.in_flight { owed.in_flight = false; continue; @@ -102,18 +108,6 @@ impl Scheduler { } } - /// Whether this scheduler can propose an entry of `kind` now. - /// - /// A vote and a `CompletionAck` answer for the vShard on every node: the - /// first one the sequencer log holds counts. So only the vShard's - /// data-group leader proposes them. A node that left the group leads none - /// of it, so an entry from its stale state never enters the log. Every - /// other replica keeps the entry owed and proposes it if it comes to lead - /// before one applies. Which entry counts then depends on the log alone. - fn may_propose(&self, kind: OwedKind) -> bool { - !kind.leader_only() || self.is_group_leader() - } - /// The registry's view of `txn_id` for this scheduler's vShard. fn participant_progress(&self, txn_id: TxnId) -> Option { self.registry @@ -289,14 +283,15 @@ mod tests { ); } - /// The registry removes a txn's entry once its outcome fired. An owed - /// entry for such a txn is settled, not proposed again. + /// The registry evicts a txn's entry once it is terminal. An owed entry + /// for such a txn is settled, not proposed again. #[tokio::test] - async fn entry_for_a_txn_whose_outcome_fired_is_settled() { + async fn entry_for_an_evicted_txn_is_settled() { let (mut scheduler, _dir) = build_test_scheduler(VSHARD); let proposer = CapturingProposer::failing_first(1); scheduler.sequencer_proposer = proposer.clone(); elect_data_group_leader(&scheduler); + scheduler.registry.set_waiterless_ttl(Duration::ZERO); let txn_id = TxnId::new(22, 0); let _outcome = scheduler .registry @@ -306,6 +301,10 @@ mod tests { txn_id, SchedulerProposal::CompletionAck { result: Vec::new() }, ); + scheduler.registry.note_verdict( + cluster_txn_id(txn_id), + nodedb_cluster::calvin::VerdictOutcome::Commit, + ); scheduler .registry .note_completion_ack(cluster_txn_id(txn_id), VSHARD); @@ -376,9 +375,15 @@ mod tests { let (mut scheduler, _dir) = build_test_scheduler(VSHARD); let proposer = CapturingProposer::accepting(); scheduler.sequencer_proposer = proposer.clone(); + elect_data_group_leader(&scheduler); let txn_id = TxnId::new(23, 0); - scheduler.propose_sequencer_entry(txn_id, SchedulerProposal::OllpMismatch); + scheduler.propose_sequencer_entry( + txn_id, + SchedulerProposal::Vote { + abort: Some(nodedb_cluster::calvin::AbortReason::PlanRejected), + }, + ); scheduler.retry_owed_sequencer_entries(); scheduler.retry_owed_sequencer_entries(); @@ -392,10 +397,11 @@ mod tests { let (mut scheduler, _dir) = build_test_scheduler(VSHARD); let proposer = CapturingProposer::failing_first(1); scheduler.sequencer_proposer = proposer.clone(); + elect_data_group_leader(&scheduler); scheduler.propose_sequencer_entry( TxnId::new(24, 0), - SchedulerProposal::RoutingFailed { - detail: "unroutable".to_string(), + SchedulerProposal::Vote { + abort: Some(nodedb_cluster::calvin::AbortReason::PlanRejected), }, ); assert!(proposer.accepted().is_empty()); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts.rs index c0601d7f3..91fc66692 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts.rs @@ -35,8 +35,9 @@ use std::sync::Arc; use nodedb_cluster::calvin::types::{SequencedTxn, TaskChunk, TxnIdWire}; use nodedb_physical::physical_plan::PhysicalPlan; -use tracing::{error, warn}; +use tracing::warn; +use super::parts_error::PartsError; use super::routing::{PlanRouting, plan_vshard_in_database}; use super::scheduler::Scheduler; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; @@ -84,12 +85,12 @@ struct Assembler { /// The task whose chunks are being joined. spanning: Option, /// Why a part cannot be read. The txn fails at dispatch. - failed: Option, + failed: Option, } impl Assembler { - fn fail(&mut self, detail: impl FnOnce() -> String) { - self.failed.get_or_insert_with(detail); + fn fail(&mut self, error: impl FnOnce() -> PartsError) { + self.failed.get_or_insert_with(error); } /// Keep `plan`, the txn's task `task` from part `index`, when it routes @@ -102,13 +103,17 @@ impl Assembler { } } PlanRouting::ControlPlaneOnly => { - self.fail(|| format!("part {index} task {task} is a control-plane-only plan")); + self.fail(|| PartsError::ControlPlaneOnly { part: index, task }); } PlanRouting::NotAWrite => { - self.fail(|| format!("part {index} task {task} is not a write")); + self.fail(|| PartsError::NotAWrite { part: index, task }); } PlanRouting::Unroutable(reason) => { - self.fail(|| format!("part {index} task {task} is unroutable: {reason}")); + self.fail(|| PartsError::Unroutable { + part: index, + task, + reason, + }); } } } @@ -117,7 +122,7 @@ impl Assembler { fn take_tasks(&mut self, index: u32, first_task: u32, plans: &[u8], vshard_id: u32) { if let Some(spanning) = &self.spanning { let task = spanning.task; - self.fail(|| format!("part {index} interrupts the chunks of task {task}")); + self.fail(|| PartsError::InterruptsChunks { part: index, task }); return; } match super::super::helpers::decode_plans(plans) { @@ -126,7 +131,10 @@ impl Assembler { self.keep_if_local(index, task, plan, vshard_id); } } - Err(error) => self.fail(|| format!("part {index} does not decode: {error}")), + Err(error) => self.fail(|| PartsError::PartUndecodable { + part: index, + source: Box::new(error), + }), } } @@ -148,7 +156,7 @@ impl Assembler { } }; if !continues { - self.fail(|| format!("part {index} does not continue the chunks of task {task}")); + self.fail(|| PartsError::BrokenChunks { part: index, task }); self.spanning = None; return; } @@ -166,12 +174,15 @@ impl Assembler { return; }; if len > whole.total_len { - self.fail(|| format!("task {task} chunks run past its {} bytes", whole.total_len)); + self.fail(|| PartsError::ChunksOverrun { + task, + total_len: whole.total_len, + }); return; } match nodedb_physical::physical_plan::wire::decode(&whole.bytes) { Ok(plan) => self.keep_if_local(index, task, plan, vshard_id), - Err(error) => self.fail(|| format!("task {task} does not decode: {error}")), + Err(source) => self.fail(|| PartsError::TaskUndecodable { task, source }), } } } @@ -253,14 +264,15 @@ impl Scheduler { /// /// A multi-part txn with every part that targets this vShard is /// assembled here. One still missing parts waits with its locks. An - /// abandoned one, and one whose part cannot be read, completes here. + /// abandoned one completes here. One whose part cannot be read votes + /// abort on its rejected plans. /// /// Returns the txn to dispatch: a single-entry txn, or a multi-part one /// with every part that targets this vShard assembled. `None` means /// nothing dispatches: the txn waits for parts, or it completed. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn gate_on_parts( &mut self, - txn: SequencedTxn, + mut txn: SequencedTxn, txn_id: TxnId, lock_owner: TxnId, ) -> Option { @@ -286,11 +298,11 @@ impl Scheduler { } let vshard_id = self.vshard_id; let assembled = match self.parts.assemblies.remove(&txn_id) { - Some(assembly) => assemble(txn, assembly, vshard_id), - None => Err("the txn reached dispatch with no assembly of its parts".to_owned()), + Some(assembly) => assemble(&mut txn, assembly, vshard_id), + None => Err(PartsError::NoAssembly), }; match assembled { - Ok(txn) => Some(txn), + Ok(()) => Some(txn), Err(detail) => { let error = crate::Error::Internal { detail: format!( @@ -298,9 +310,7 @@ impl Scheduler { txn_id.epoch, txn_id.position, self.vshard_id ), }; - error!(vshard_id = self.vshard_id, %error, "calvin scheduler: part unreadable"); - self.propose_routing_failure(txn_id, &error); - self.on_unpending_txn_complete(txn_id, lock_owner); + self.reject_plan(txn, txn_id, lock_owner, &error); None } } @@ -380,12 +390,8 @@ impl Scheduler { /// /// Reads the parts in index order, so the result depends only on their /// contents. Every replica that holds the same parts assembles the same -/// plans, or fails alike. -fn assemble( - mut txn: SequencedTxn, - assembly: Assembly, - vshard_id: u32, -) -> Result { +/// plans, or fails alike. A failure leaves `txn` unchanged. +fn assemble(txn: &mut SequencedTxn, assembly: Assembly, vshard_id: u32) -> Result<(), PartsError> { let mut assembler = Assembler { database_id: assembly.database_id, local: Vec::new(), @@ -404,12 +410,11 @@ fn assemble( return Err(failed); } if let Some(spanning) = assembler.spanning { - return Err(format!( - "task {} stopped at byte {} of {}", - spanning.task, - spanning.bytes.len(), - spanning.total_len - )); + return Err(PartsError::Unfinished { + task: spanning.task, + received: spanning.bytes.len(), + total_len: spanning.total_len, + }); } let mut local = assembler.local; local.sort_by_key(|(task, _)| *task); @@ -419,26 +424,35 @@ fn assemble( .map(|(local_index, _)| local_index) .collect(); let plans: Vec<&PhysicalPlan> = local.iter().map(|(_, plan)| plan).collect(); - txn.tx_class.plans = - zerompk::to_msgpack_vec(&plans).map_err(|e| format!("local plans do not encode: {e}"))?; + let bytes = zerompk::to_msgpack_vec(&plans) + .map_err(|source| PartsError::PlansDoNotEncode { source })?; + txn.tx_class.plans = bytes; txn.tx_class.body_plans = body_plans; - Ok(txn) + Ok(()) } #[cfg(test)] mod tests { - use nodedb_cluster::calvin::CalvinCompletionRegistry; use nodedb_cluster::calvin::types::{ MultiPartPlans, PartStreamId, SchedulerInput, VShardParts, }; + use nodedb_cluster::calvin::{AbortReason, CalvinCompletionRegistry, SequencerEntry}; use nodedb_physical::physical_plan::DocumentOp; use nodedb_physical::physical_plan::wire as plan_wire; use nodedb_types::QualifiedCollection; use super::*; + use crate::bridge::dispatch::CoreChannelDataSide; + use crate::bridge::envelope::Status; + use crate::control::cluster::calvin::scheduler::driver::core::owed::OwedKind; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ + CapturingProposer, elect_data_group_leader, + }; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ - build_test_scheduler_with_data_side, make_local_write_txn, test_coll_vshard, + build_test_scheduler_with_data_side, make_local_write_txn, staged_response, + test_coll_vshard, }; + use crate::control::cluster::calvin::scheduler::driver::types::CommitState; use crate::types::DatabaseId; fn truncate_plan() -> PhysicalPlan { @@ -514,6 +528,107 @@ mod tests { plan_wire::encode_batch(&vec![truncate_plan()]).expect("encode one truncate") } + /// A scheduler on the `test_coll` vShard that leads its data group and + /// captures its proposals. + fn leading_scheduler() -> ( + Scheduler, + tempfile::TempDir, + CoreChannelDataSide, + Arc, + ) { + let (mut scheduler, dir, data_side) = build_test_scheduler_with_data_side( + test_coll_vshard(), + CalvinCompletionRegistry::new_detached(), + ); + let proposer = CapturingProposer::accepting(); + scheduler.sequencer_proposer = proposer.clone(); + elect_data_group_leader(&scheduler); + (scheduler, dir, data_side, proposer) + } + + /// Whether a truncate of `test_coll` reached the Data Plane. Drains the + /// request ring. + fn truncate_reached_data_plane(data_side: &mut CoreChannelDataSide) -> bool { + let mut reached = false; + while let Ok(request) = data_side.request_rx.try_pop() { + reached |= request.inner.plan == truncate_plan(); + } + reached + } + + /// `txn_id` is rejected before it stages. It waits in `pending` for its + /// verdict with a stage error, owes a `PlanRejected` vote, and holds its + /// locks: `next`, on the same key, waits behind it. At the abort verdict + /// it drops, marks its position applied, and frees its locks, so `next` + /// stages. + async fn rejected_then_dropped_at_its_verdict( + scheduler: &mut Scheduler, + data_side: &mut CoreChannelDataSide, + proposer: &CapturingProposer, + txn_id: TxnId, + next: SequencedTxn, + ) { + let pending = scheduler + .pending + .get(&txn_id) + .expect("waits for its verdict"); + assert_eq!(pending.commit_state, CommitState::AwaitingVerdict); + assert!(pending.stage_error.is_some(), "the rejection is recorded"); + assert!(!scheduler.parts.is_awaiting(txn_id)); + assert!(scheduler.owed.contains_key(&(txn_id, OwedKind::Vote))); + assert!(proposer.accepted().contains(&SequencerEntry::AbortVote { + epoch: txn_id.epoch, + position: txn_id.position, + vshard: test_coll_vshard(), + reason: AbortReason::PlanRejected, + })); + assert!(!scheduler.applied.is_applied(txn_id.epoch, txn_id.position)); + + let next_id = TxnId::new(next.epoch, next.position); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(next))); + assert!( + scheduler.blocked.contains_key(&next_id), + "the rejected txn holds its locks" + ); + + scheduler.registry.note_verdict( + nodedb_cluster::calvin::TxnId::new(txn_id.epoch, txn_id.position), + nodedb_cluster::calvin::VerdictOutcome::Abort(AbortReason::PlanRejected), + ); + scheduler.resume_on_verdict(txn_id, false); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingResolve { + committed: false, + redo_lsn: None + }) + ); + assert!( + !truncate_reached_data_plane(data_side), + "no plan of the rejected txn stages" + ); + assert!(!scheduler.applied.is_applied(txn_id.epoch, txn_id.position)); + + scheduler + .finish_resolved_commit(txn_id, staged_response(Status::Ok, None), false, None) + .await; + assert!(!scheduler.pending.contains_key(&txn_id)); + assert!(scheduler.applied.is_applied(txn_id.epoch, txn_id.position)); + assert!( + proposer.accepted().iter().any(|entry| matches!( + entry, + SequencerEntry::CompletionAck { epoch, position, .. } + if *epoch == txn_id.epoch && *position == txn_id.position + )), + "the dropped txn acks" + ); + assert!(!scheduler.blocked.contains_key(&next_id)); + assert!( + scheduler.pending.contains_key(&next_id), + "the freed locks let the next txn stage" + ); + } + /// A txn whose locks are granted before its parts holds them and waits. /// It stages once the last part that targets this vShard arrived, with /// every local task of every part. @@ -614,13 +729,11 @@ mod tests { assert_eq!(plans, vec![truncate_plan()]); } - /// A chunk that skips bytes fails the txn: nothing stages. + /// A chunk that skips bytes rejects the txn before it stages. It votes + /// abort and drops at the verdict. #[tokio::test] - async fn a_chunk_that_skips_bytes_stages_nothing() { - let (mut scheduler, _dir, _data_side) = build_test_scheduler_with_data_side( - test_coll_vshard(), - CalvinCompletionRegistry::new_detached(), - ); + async fn a_chunk_that_skips_bytes_is_rejected_before_it_stages() { + let (mut scheduler, _dir, mut data_side, proposer) = leading_scheduler(); let txn_id = TxnId::new(8, 0); let task = zerompk::to_msgpack_vec(&truncate_plan()).expect("encode one task"); let total = task.len() as u64; @@ -628,30 +741,35 @@ mod tests { scheduler.process_scheduler_input(chunk_part(8, 0, 0, &task[..2], 0, total)); scheduler.process_scheduler_input(chunk_part(8, 1, 0, &task[3..], 3, total)); - assert!(!scheduler.pending.contains_key(&txn_id), "nothing staged"); - assert!(scheduler.applied.is_applied(8, 0), "the txn completed"); + rejected_then_dropped_at_its_verdict( + &mut scheduler, + &mut data_side, + &proposer, + txn_id, + plain_write(8, 1), + ) + .await; } - /// A late part that does not decode fails the txn before it stages: the - /// earlier part's task never stages, and the txn's locks are released. + /// A late part that does not decode rejects the txn before it stages: + /// the earlier part's task never reaches the Data Plane. The txn votes + /// abort and drops at the verdict. #[tokio::test] - async fn a_late_part_that_fails_stages_nothing_of_the_earlier_parts() { - let (mut scheduler, _dir, _data_side) = build_test_scheduler_with_data_side( - test_coll_vshard(), - CalvinCompletionRegistry::new_detached(), - ); + async fn a_late_part_that_fails_rejects_the_txn_before_the_earlier_parts_stage() { + let (mut scheduler, _dir, mut data_side, proposer) = leading_scheduler(); let txn_id = TxnId::new(3, 0); scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(two_part_header(3, 0)))); scheduler.process_scheduler_input(part(3, 0, 0, one_truncate())); scheduler.process_scheduler_input(part(3, 0, 1, vec![0xc1, 0xff, 0x00])); - assert!(!scheduler.pending.contains_key(&txn_id), "nothing staged"); - assert!(!scheduler.parts.is_awaiting(txn_id)); - assert!(scheduler.applied.is_applied(3, 0), "the txn completed"); - - // Its locks are free: the next txn on the same key stages at once. - scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(plain_write(3, 1)))); - assert!(scheduler.pending.contains_key(&TxnId::new(3, 1))); + rejected_then_dropped_at_its_verdict( + &mut scheduler, + &mut data_side, + &proposer, + txn_id, + plain_write(3, 1), + ) + .await; } /// An abandoned txn holding its locks completes at once and frees them. diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts_error.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts_error.rs new file mode 100644 index 000000000..9e3a39ef4 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts_error.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Why a multi-part txn's parts do not assemble into local plans. + +use nodedb_physical::physical_plan::wire::WireError; + +/// A part set that does not assemble. Every replica holding the same parts +/// fails with the same error, so the txn is rejected alike everywhere. +#[derive(Debug, thiserror::Error)] +pub(in crate::control::cluster::calvin::scheduler::driver::core) enum PartsError { + #[error("the txn reached dispatch with no assembly of its parts")] + NoAssembly, + #[error("part {part} task {task} is a control-plane-only plan")] + ControlPlaneOnly { part: u32, task: u32 }, + #[error("part {part} task {task} is not a write")] + NotAWrite { part: u32, task: u32 }, + #[error("part {part} task {task} is unroutable: {reason}")] + Unroutable { + part: u32, + task: u32, + reason: &'static str, + }, + #[error("part {part} interrupts the chunks of task {task}")] + InterruptsChunks { part: u32, task: u32 }, + #[error("part {part} does not decode: {source}")] + PartUndecodable { + part: u32, + source: Box, + }, + #[error("part {part} does not continue the chunks of task {task}")] + BrokenChunks { part: u32, task: u32 }, + #[error("task {task} chunks run past its {total_len} bytes")] + ChunksOverrun { task: u32, total_len: u64 }, + #[error("task {task} does not decode: {source}")] + TaskUndecodable { task: u32, source: WireError }, + #[error("task {task} stopped at byte {received} of {total_len}")] + Unfinished { + task: u32, + received: usize, + total_len: u64, + }, + #[error("local plans do not encode: {source}")] + PlansDoNotEncode { source: zerompk::Error }, +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs index d8b9e8818..f83ff80e5 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs @@ -271,8 +271,9 @@ impl Scheduler { /// /// Releases the lock-table owner recorded in its `pending` entry. A txn /// with no `pending` entry has already completed, so this logs and - /// releases nothing. A txn that fails before it enters `pending` uses - /// [`Self::on_unpending_txn_complete`] instead. + /// releases nothing. A multi-part txn whose parts the sequencer abandoned + /// never enters `pending`, and uses [`Self::on_unpending_txn_complete`] + /// instead. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn on_txn_complete( &mut self, txn_id: TxnId, @@ -293,8 +294,9 @@ impl Scheduler { self.release_and_mark_applied(txn_id, pending.lock_owner); } - /// Complete a txn that failed before it entered `pending`, releasing the - /// locks held under `lock_owner`. + /// Complete a txn that never entered `pending`, releasing the locks held + /// under `lock_owner`. Only a txn whose stored verdict is already + /// `Abort(PartsLost)` completes this way. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn on_unpending_txn_complete( &mut self, txn_id: TxnId, diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs index 41a0c0093..6e8815b6d 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs @@ -52,6 +52,10 @@ impl Scheduler { } /// Check all pending dependent barriers for timeout. + /// + /// A timed-out txn never staged here. It parks on the commit barrier + /// with a `ParticipantError` abort vote and keeps its locks. Its position + /// is marked applied only once the abort verdict drops it. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn check_dependent_barrier_timeouts( &mut self, ) { @@ -70,16 +74,116 @@ impl Scheduler { epoch = txn_id.epoch, position = txn_id.position, still_waiting = ?barrier.waiting_for, - "calvin: dependent-read barrier timed out; releasing locks" + "calvin: dependent-read barrier timed out; voting abort" ); self.metrics.record_executor_error(); self.metrics.record_infra_abort( crate::control::cluster::calvin::scheduler::metrics::infra_abort_reason::PASSIVE_PARTICIPANT_TIMEOUT, ); - // A barrier txn never entered `pending`: release its locks - // under the owner the barrier recorded. - self.on_unpending_txn_complete(txn_id, barrier.lock_owner); + let stage_error = format!( + "dependent-read barrier timed out waiting for passive vShards {:?}", + barrier.waiting_for + ); + self.park_unstaged( + barrier.txn, + txn_id, + barrier.lock_owner, + nodedb_cluster::calvin::AbortReason::ParticipantError, + stage_error, + ); } } } } + +#[cfg(test)] +mod tests { + use std::collections::{BTreeMap, BTreeSet}; + use std::time::{Duration, Instant}; + + use nodedb_cluster::calvin::{ + AbortReason, CalvinCompletionRegistry, ParticipantVote, SequencerEntry, VerdictOutcome, + }; + + use super::*; + use crate::control::cluster::calvin::scheduler::driver::barrier::PendingDependentBarrier; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ + CapturingProposer, elect_data_group_leader, + }; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler_with_data_side, make_sequenced_txn, + }; + use crate::control::cluster::calvin::scheduler::driver::types::CommitState; + + const VSHARD: u32 = 7; + + /// A timed-out barrier votes abort and parks. Its vote completes the + /// tally into an abort verdict, and the position stays unapplied until + /// that verdict drops the txn. + #[tokio::test] + async fn a_timed_out_barrier_votes_a_complete_abort_tally() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, _data_side) = + build_test_scheduler_with_data_side(VSHARD, std::sync::Arc::clone(®istry)); + let proposer = CapturingProposer::accepting(); + scheduler.sequencer_proposer = proposer.clone(); + elect_data_group_leader(&scheduler); + let txn_id = TxnId::new(40, 0); + let cluster_txn = nodedb_cluster::calvin::TxnId::new(40, 0); + registry.seed_expected(cluster_txn, 1); + scheduler.dependent_barrier.insert( + txn_id, + PendingDependentBarrier { + txn: make_sequenced_txn(40, 0), + lock_owner: txn_id, + waiting_for: BTreeSet::from([9]), + received: BTreeMap::new(), + // no-determinism: test-only deadline already passed. + timeout_at: Instant::now() - Duration::from_millis(1), + }, + ); + + scheduler.check_dependent_barrier_timeouts(); + assert!(scheduler.dependent_barrier.is_empty()); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingVerdict) + ); + assert!(!scheduler.applied.is_applied(40, 0)); + assert_eq!( + proposer.accepted(), + vec![SequencerEntry::AbortVote { + epoch: 40, + position: 0, + vshard: VSHARD, + reason: AbortReason::ParticipantError, + }] + ); + + registry.note_vote( + cluster_txn, + VSHARD, + ParticipantVote::Abort(AbortReason::ParticipantError), + ); + assert_eq!( + registry.drain_unproposed_verdicts(), + vec![( + cluster_txn, + VerdictOutcome::Abort(AbortReason::ParticipantError) + )] + ); + registry.note_verdict( + cluster_txn, + VerdictOutcome::Abort(AbortReason::ParticipantError), + ); + scheduler.resume_on_verdict(txn_id, false); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingResolve { + committed: false, + redo_lsn: None + }) + ); + assert!(!scheduler.applied.is_applied(40, 0)); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs index 002090c4a..35e5148fb 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs @@ -116,6 +116,10 @@ pub struct Scheduler { /// Shared mirror of `applied`, read by authorization coverage. pub(in crate::control::cluster::calvin::scheduler::driver::core) applied_mirror: Arc, + /// This scheduler's caught-up entry, read by the startup readiness gate. + /// Set once the watermark reaches `rebuild_target_epoch`. + pub(in crate::control::cluster::calvin::scheduler::driver::core) caught_up: + crate::control::cluster::calvin::scheduler::CaughtUpHandle, /// Scheduler configuration. pub(in crate::control::cluster::calvin::scheduler::driver::core) config: SchedulerConfig, /// Metrics. @@ -248,11 +252,9 @@ impl Scheduler { let completion_cap = config.channel_capacity; let (completion_tx, completion_rx) = mpsc::channel(completion_cap); - let applied_mirror = shared.authorization_fence.calvin_mirrors().register( - vshard_id, - fully_applied_epoch, - &applied_tail, - ); + let mirrors = shared.authorization_fence.calvin_mirrors(); + let applied_mirror = mirrors.register(vshard_id, fully_applied_epoch, &applied_tail); + let caught_up = mirrors.caught_up().register(vshard_id); // A backup's cut waits on every scheduler this node runs. shared.calvin.cuts.register(vshard_id); @@ -264,7 +266,7 @@ impl Scheduler { .unwrap_or_else(|p| p.into_inner()) .capacity_freed(); - Self { + let scheduler = Self { vshard_id, receiver, shared, @@ -280,6 +282,7 @@ impl Scheduler { applied: AppliedGate::new(fully_applied_epoch, applied_tail), cut_floors: Default::default(), applied_mirror, + caught_up, rebuild_target_epoch, max_input_epoch: 0, config, @@ -296,7 +299,10 @@ impl Scheduler { metadata_hold: None, parts: Default::default(), install_gate, - } + }; + // A scheduler with nothing to rebuild reports caught up at once. + scheduler.publish_caught_up(); + scheduler } /// Whether the scheduler has caught up to the rebuild target epoch. @@ -329,6 +335,21 @@ impl Scheduler { fully_applied >= self.rebuild_target_epoch } + /// Report this scheduler caught up to the readiness gate, once + /// [`Self::is_caught_up`] holds. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn publish_caught_up(&self) { + if self.caught_up.is_caught_up() || !self.is_caught_up() { + return; + } + self.caught_up.mark_caught_up(); + tracing::info!( + vshard_id = self.vshard_id, + rebuild_target_epoch = self.rebuild_target_epoch, + fully_applied_epoch = self.applied.fully_applied_epoch(), + "calvin scheduler caught up to its rebuild target" + ); + } + /// Publish an advanced fully-applied watermark to the metrics gauge and the /// shared cross-shard snapshot anchor. /// @@ -349,6 +370,7 @@ impl Scheduler { .fetch_max(watermark, std::sync::atomic::Ordering::Release); // A marker passes once every epoch delivered before it folded. self.report_passed_cuts(); + self.publish_caught_up(); } } @@ -423,4 +445,57 @@ mod tests { "no rebuild target (greenfield node) must report caught-up" ); } + + fn lagging(scheduler: &Scheduler) -> Vec { + scheduler + .shared + .authorization_fence + .calvin_mirrors() + .caught_up() + .lagging() + } + + /// A scheduler with a real rebuild target holds the readiness gate until + /// its published watermark reaches that target. + #[tokio::test] + async fn a_scheduler_lags_until_its_watermark_reaches_the_rebuild_target() { + let (mut scheduler, _dir) = build_test_scheduler(4); + scheduler.rebuild_target_epoch = 3; + assert_eq!(lagging(&scheduler), vec![4]); + + scheduler.applied = AppliedGate::new(2, BTreeSet::new()); + scheduler.publish_watermark(2); + assert_eq!(lagging(&scheduler), vec![4], "epoch 2 is below the target"); + + scheduler.applied = AppliedGate::new(3, BTreeSet::new()); + scheduler.publish_watermark(3); + assert!(lagging(&scheduler).is_empty()); + } + + /// With no Calvin history there is nothing to rebuild, so the scheduler + /// reports caught up without a watermark. + #[tokio::test] + async fn a_scheduler_with_nothing_to_rebuild_reports_caught_up() { + let (mut scheduler, _dir) = build_test_scheduler(4); + scheduler.rebuild_target_epoch = NOT_YET_APPLIED_EPOCH; + scheduler.publish_caught_up(); + assert!(lagging(&scheduler).is_empty()); + } + + /// A stopped scheduler no longer holds the readiness gate. + #[tokio::test] + async fn a_dropped_scheduler_leaves_the_readiness_registry() { + let (scheduler, _dir) = build_test_scheduler(4); + let shared = Arc::clone(&scheduler.shared); + assert_eq!(lagging(&scheduler), vec![4]); + drop(scheduler); + assert!( + shared + .authorization_fence + .calvin_mirrors() + .caught_up() + .lagging() + .is_empty() + ); + } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs index a8ca94fc2..d2a888822 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs @@ -438,7 +438,7 @@ pub(super) fn staged_pending(txn: SequencedTxn, txn_id: TxnId) -> PendingTxn { has_primary_write: true, has_returning: false, change_sets: Vec::new(), - commit_state: Some(CommitState::Staged), + commit_state: CommitState::Staged, verdict_deadline: None, stage_error: None, redo_records: None, @@ -491,7 +491,7 @@ pub(super) fn scheduler_with_pending( ) -> (Scheduler, tempfile::TempDir) { let (mut scheduler, dir) = build_test_scheduler(7); let mut pending = staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id); - pending.commit_state = Some(state); + pending.commit_state = state; scheduler.pending.insert(txn_id, pending); (scheduler, dir) } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/write_version_record.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/write_version_record.rs index 0d27cb9e8..2a68d70c2 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/write_version_record.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/write_version_record.rs @@ -11,13 +11,27 @@ //! write-version recorder at that LSN — the same shard-local WAL-LSN space //! the single-shard fast path and read watermarks use. +use super::super::types::CommitState; use super::deferred::{DispatchOutcome, DispatchStep}; +use super::halt::{HaltReason, HaltStep}; use super::scheduler::Scheduler; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; use crate::types::Lsn; use nodedb_physical::physical_plan::PhysicalPlan; use nodedb_physical::physical_plan::meta::MetaOp; +/// Result of [`Scheduler::record_calvin_write_versions`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::control::cluster::calvin::scheduler::driver::core) enum VersionRecord { + /// The dispatcher accepted the record. + Sent, + /// The record waits for capacity. The txn stays pending until it is sent. + Parked, + /// The record cannot be built or sent. The scheduler halted, and the txn + /// stays pending and unapplied. + Halted, +} + impl Scheduler { /// Record the per-key write versions of a just-committed Calvin /// transaction's locally-applied write plans at its committed WAL @@ -27,41 +41,34 @@ impl Scheduler { /// this vShard's core with `applied_lsn` stamped on the request envelope's /// `wal_lsn`; the core funnels the plans through the shared write-version /// recorder at that LSN. The recorded version therefore lands in the same - /// WAL-LSN space as fast-path writes, so a later read-set validation against - /// these keys is not a false-Valid serializability hole. + /// WAL-LSN space as fast-path writes. + /// + /// The record feeds read-set validation. A later txn's stage checks each + /// read against these versions. A missing version reports a stale read as + /// valid. So the txn keeps its locks until the record is sent. The core + /// applies requests in dispatch order, so the record lands before the + /// stage of any txn that waits on these locks. The response is drained + /// and discarded. /// - /// Fire-and-forget: the recorded version is not needed to complete the - /// transaction, so the response is drained and discarded. A brief index-lag - /// window before the record op lands is harmless — nothing enforces read-set - /// validation against these versions yet. A record refused at capacity is - /// parked and re-sent once capacity frees, never dropped. A decode or - /// routing failure, or a terminal dispatch refusal, leaves the version - /// unrecorded and never blocks the commit. + /// A record refused at capacity is parked and re-sent once capacity + /// frees. A decode or routing error, or a terminal dispatch refusal, + /// halts the scheduler. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn record_calvin_write_versions( &mut self, txn_id: TxnId, applied_lsn: Lsn, - ) { + ) -> VersionRecord { let epoch = txn_id.epoch; let position = txn_id.position; let Some(pending) = self.pending.get(&txn_id) else { - return; + return self.halt_version_record(txn_id, "the txn has no pending entry".to_string()); }; let tenant_id = pending.txn.tx_class.tenant_id; let database_id = pending.txn.tx_class.database_id; let plans = match super::super::helpers::decode_plans(&pending.txn.tx_class.plans) { Ok(p) => p, - Err(e) => { - tracing::warn!( - vshard_id = self.vshard_id, - epoch, - position, - error = %e, - "calvin: write-version recording skipped — plan decode failed" - ); - return; - } + Err(e) => return self.halt_version_record(txn_id, format!("plan decode: {e}")), }; // The locally-applied slice this vShard committed. Empty for a // validate-only READ participant (no local writes) — the recorder then @@ -72,14 +79,7 @@ impl Scheduler { let local = match self.local_calvin_plans(plans, database_id, epoch, position) { Ok(p) => p, Err(e) => { - tracing::warn!( - vshard_id = self.vshard_id, - epoch, - position, - error = %e, - "calvin: write-version recording skipped — local plan routing failed" - ); - return; + return self.halt_version_record(txn_id, format!("local plan routing: {e}")); } }; @@ -93,14 +93,42 @@ impl Scheduler { let request = self.build_exempt_request(request_id, tenant_id, database_id, plan, Some(applied_lsn)); - // The request carries everything a re-send needs, so a parked record - // outlives the txn's `pending` entry. - if let DispatchOutcome::Failed(error) = - self.dispatch_sequenced(txn_id, DispatchStep::WriteVersionRecord, request) - { - self.fail_dispatch_step(txn_id, DispatchStep::WriteVersionRecord, error); + match self.dispatch_sequenced(txn_id, DispatchStep::WriteVersionRecord, request) { + DispatchOutcome::Sent => VersionRecord::Sent, + DispatchOutcome::Deferred => VersionRecord::Parked, + DispatchOutcome::Failed(error) => { + self.fail_dispatch_step(txn_id, DispatchStep::WriteVersionRecord, error); + VersionRecord::Halted + } + } + } + + /// Complete `txn_id` once its parked write-version record is sent, when + /// its commit tail already ran. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn complete_after_version_record( + &mut self, + txn_id: TxnId, + ) { + let held = self + .pending + .get(&txn_id) + .is_some_and(|pending| pending.commit_state == CommitState::AwaitingVersionRecord); + if held { + self.metrics.record_completed(); + self.on_txn_complete(txn_id); } } + + /// Halt on a write-version record that cannot be built. + fn halt_version_record(&mut self, txn_id: TxnId, error: String) -> VersionRecord { + self.halt_apply( + txn_id, + HaltReason::WriteVersionRecordFailed, + HaltStep::WriteVersionRecord, + error, + ); + VersionRecord::Halted + } } #[cfg(test)] @@ -111,10 +139,11 @@ mod tests { use nodedb_types::TenantId; use super::*; + use crate::control::cluster::calvin::scheduler::driver::core::intake::IntakeClosure; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ - await_data_plane_request, build_test_scheduler_with_data_side, fill_tenant_inflight, - make_validate_only_txn, release_filler, spawn_scheduler_loop, staged_pending, - test_coll_vshard, + await_data_plane_request, begin_data_plane_drain, build_test_scheduler, + build_test_scheduler_with_data_side, fill_tenant_inflight, make_validate_only_txn, + release_filler, spawn_scheduler_loop, staged_pending, test_coll_vshard, }; /// Once a Data Plane response frees tenant capacity, a write-version @@ -132,7 +161,10 @@ mod tests { let shared = Arc::clone(&scheduler.shared); let fillers = fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); - scheduler.record_calvin_write_versions(txn_id, Lsn::new(42)); + assert_eq!( + scheduler.record_calvin_write_versions(txn_id, Lsn::new(42)), + VersionRecord::Parked + ); let running = spawn_scheduler_loop(scheduler); release_filler(&shared, &mut data_side, fillers[0]); @@ -150,4 +182,125 @@ mod tests { "the refused write-version record must reach the Data Plane once capacity frees" ); } + + /// A write-version record refused terminally halts the scheduler. The + /// txn stays pending and unapplied, and intake closes. + #[tokio::test] + async fn terminal_write_version_record_refusal_halts_and_holds_the_txn() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, _data_side) = + build_test_scheduler_with_data_side(test_coll_vshard(), registry); + let txn_id = TxnId::new(22, 0); + scheduler.pending.insert( + txn_id, + staged_pending(make_validate_only_txn(22, 0), txn_id), + ); + begin_data_plane_drain(&scheduler.shared); + + assert_eq!( + scheduler.record_calvin_write_versions(txn_id, Lsn::new(42)), + VersionRecord::Halted + ); + + assert!(!scheduler.applied.is_applied(22, 0)); + assert!(scheduler.pending.contains_key(&txn_id)); + assert_eq!( + scheduler.apply_halt().map(|h| h.report.step), + Some("write_version_record") + ); + assert_eq!(scheduler.intake_closure(), Some(IntakeClosure::ApplyHalted)); + } + + /// A parked write-version record refused terminally on re-send halts the + /// scheduler, and its held txn stays pending and unapplied. + #[tokio::test] + async fn parked_write_version_record_refused_on_resend_holds_the_txn() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, mut data_side) = + build_test_scheduler_with_data_side(test_coll_vshard(), registry); + let txn_id = TxnId::new(23, 0); + scheduler.pending.insert( + txn_id, + staged_pending(make_validate_only_txn(23, 0), txn_id), + ); + let shared = Arc::clone(&scheduler.shared); + fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); + assert_eq!( + scheduler.record_calvin_write_versions(txn_id, Lsn::new(42)), + VersionRecord::Parked + ); + if let Some(pending) = scheduler.pending.get_mut(&txn_id) { + pending.commit_state = CommitState::AwaitingVersionRecord; + } + + begin_data_plane_drain(&shared); + scheduler.redispatch_deferred(); + + assert!(!scheduler.applied.is_applied(23, 0)); + assert!(scheduler.pending.contains_key(&txn_id)); + assert_eq!( + scheduler.apply_halt().map(|h| h.report.step), + Some("write_version_record") + ); + assert!(!scheduler.resends_deferred()); + } + + /// A txn held for its parked write-version record completes once the + /// record is sent. + #[tokio::test] + async fn held_txn_completes_once_its_write_version_record_is_sent() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, mut data_side) = + build_test_scheduler_with_data_side(test_coll_vshard(), registry); + let txn_id = TxnId::new(24, 0); + scheduler.pending.insert( + txn_id, + staged_pending(make_validate_only_txn(24, 0), txn_id), + ); + let shared = Arc::clone(&scheduler.shared); + let fillers = fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); + assert_eq!( + scheduler.record_calvin_write_versions(txn_id, Lsn::new(42)), + VersionRecord::Parked + ); + if let Some(pending) = scheduler.pending.get_mut(&txn_id) { + pending.commit_state = CommitState::AwaitingVersionRecord; + } + assert!( + !scheduler.applied.is_applied(24, 0), + "a parked record holds the position unapplied" + ); + + release_filler(&shared, &mut data_side, fillers[0]); + scheduler.redispatch_deferred(); + + assert!(!scheduler.has_deferred_dispatch()); + assert!(!scheduler.pending.contains_key(&txn_id)); + assert!(scheduler.applied.is_applied(24, 0)); + assert!(!scheduler.is_apply_halted()); + } + + /// Plans that do not decode leave no record to send. The scheduler + /// halts, and the txn stays pending and unapplied. + #[tokio::test] + async fn undecodable_plans_halt_the_write_version_record() { + let (mut scheduler, _dir) = build_test_scheduler(test_coll_vshard()); + let txn_id = TxnId::new(25, 0); + let mut pending = staged_pending(make_validate_only_txn(25, 0), txn_id); + // 0xc1 is a reserved MessagePack marker, so no plan batch decodes. + pending.txn.tx_class.plans = vec![0xc1]; + scheduler.pending.insert(txn_id, pending); + + assert_eq!( + scheduler.record_calvin_write_versions(txn_id, Lsn::new(42)), + VersionRecord::Halted + ); + + assert!(!scheduler.applied.is_applied(25, 0)); + assert!(scheduler.pending.contains_key(&txn_id)); + assert_eq!( + scheduler.apply_halt().map(|h| h.reason), + Some(HaltReason::WriteVersionRecordFailed) + ); + } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs index f3cda4a1a..7e08b9fe1 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs @@ -46,13 +46,12 @@ pub(super) struct PendingTxn { /// Participant-local Control-Plane change manifests. Consumed once after /// durable COMMIT apply; graph dual-home replicas are excluded at capture. pub change_sets: Vec, - /// Commit-resolution state for a static-set Calvin txn. + /// Commit-resolution state of the txn. /// - /// `Some(CommitState::Staged)` for a txn dispatched, or parked for re-send, - /// via the validate-and-stage path: its first executor response carries the local - /// commit vote and drives a flush-or-drop before the commit tail runs. - /// `None` for dependent/active txns, which apply directly. - pub commit_state: Option, + /// Every dispatch stages the txn and starts in [`CommitState::Staged`]. + /// The first executor response carries the local commit vote. It drives a + /// flush-or-drop before the commit tail runs. + pub commit_state: CommitState, /// Stall deadline for a txn parked in [`CommitState::AwaitingVerdict`]. /// /// `Some(instant)` only while parked: if the deadline passes with the @@ -213,6 +212,10 @@ pub(in crate::control::cluster::calvin::scheduler::driver) enum CommitState { committed: bool, redo_lsn: Option, }, + /// The commit tail ran, and the write-version record waits for capacity. + /// The txn completes once the record is sent. Its locks hold until then, + /// so no later txn validates a read against the missing versions. + AwaitingVersionRecord, } /// A transaction that is blocked on lock acquisition. diff --git a/nodedb/src/control/cluster/calvin/scheduler/metrics.rs b/nodedb/src/control/cluster/calvin/scheduler/metrics.rs index 3b5d4e62d..b87e85753 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/metrics.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/metrics.rs @@ -79,7 +79,7 @@ pub struct SchedulerMetrics { /// Owed sequencer entries re-proposed because their effect was not yet /// applied, by kind. Indexes are the constants in /// [`sequencer_propose_kind`]. - pub sequencer_propose_retry_counts: [AtomicU64; 4], + pub sequencer_propose_retry_counts: [AtomicU64; sequencer_propose_kind::LABELS.len()], } /// Reason codes for `nodedb_calvin_infra_abort_total`. @@ -127,6 +127,7 @@ pub mod apply_halt_reason { pub const IDENTITY_BIND_FAILED: usize = 6; pub const WAL_APPEND_FAILED: usize = 7; pub const METADATA_GROUP_GONE: usize = 8; + pub const WRITE_VERSION_RECORD_FAILED: usize = 9; pub const LABELS: &[&str] = &[ "draining", @@ -138,6 +139,7 @@ pub mod apply_halt_reason { "identity_bind_failed", "wal_append_failed", "metadata_group_gone", + "write_version_record_failed", ]; } @@ -145,10 +147,8 @@ pub mod apply_halt_reason { pub mod sequencer_propose_kind { pub const VOTE: usize = 0; pub const COMPLETION_ACK: usize = 1; - pub const OLLP_MISMATCH: usize = 2; - pub const ROUTING_FAILED: usize = 3; - pub const LABELS: &[&str] = &["vote", "completion_ack", "ollp_mismatch", "routing_failed"]; + pub const LABELS: &[&str] = &["vote", "completion_ack"]; } impl SchedulerMetrics { diff --git a/nodedb/src/control/cluster/calvin/scheduler/metrics_flow.rs b/nodedb/src/control/cluster/calvin/scheduler/metrics_flow.rs index 019437c27..2117749e3 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/metrics_flow.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/metrics_flow.rs @@ -128,8 +128,5 @@ mod tests { assert!(out.contains( "nodedb_calvin_sequencer_propose_retry_total{vshard=\"3\",kind=\"completion_ack\"} 1" )); - assert!(out.contains( - "nodedb_calvin_sequencer_propose_retry_total{vshard=\"3\",kind=\"routing_failed\"} 0" - )); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/mod.rs index 26126e309..be74f239e 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/mod.rs @@ -2,6 +2,7 @@ pub mod applied_gate; pub mod applied_mirror; +pub mod caught_up; pub mod cut_floor; pub mod driver; pub mod lock; @@ -11,6 +12,7 @@ pub mod recovery; pub use applied_gate::AppliedGate; pub use applied_mirror::{AppliedMirror, AppliedMirrors}; +pub use caught_up::{CaughtUpHandle, CaughtUpRegistry}; pub use driver::{ CalvinReadResultProposal, RaftSequencerProposer, ReadResultEvent, Scheduler, SchedulerConfig, SchedulerParams, SequencerProposer, propose_calvin_read_result, diff --git a/nodedb/src/control/cluster/sequencer_snapshot.rs b/nodedb/src/control/cluster/sequencer_snapshot.rs index 96c7bdec6..f25219720 100644 --- a/nodedb/src/control/cluster/sequencer_snapshot.rs +++ b/nodedb/src/control/cluster/sequencer_snapshot.rs @@ -104,28 +104,44 @@ impl SequencerSnapshotStore { /// The log then delivers only the entries after the file's index again: /// the state machine skips every entry at or below the index it /// restored. + /// + /// A boundary above 0 with no usable file leaves the entries through the + /// boundary unrecoverable. The state machine then records its history as + /// unknown, so its leader never seeds an epoch from partial state. pub fn restore_at_boot( &self, multi_raft: &nodedb_cluster::multi_raft::MultiRaft, ) -> crate::Result { + let Ok((_, boundary, _)) = multi_raft.snapshot_metadata(SEQUENCER_GROUP_ID) else { + return Ok(false); + }; + let snapshot = self.read_kept()?; + let mut state_machine = self.state_machine.lock().unwrap_or_else(|p| p.into_inner()); + match snapshot { + Some(snapshot) if snapshot.applied_index() >= boundary => { + state_machine.restore_snapshot(snapshot); + Ok(true) + } + _ => { + if boundary > 0 { + state_machine.mark_history_unknown(); + } + Ok(false) + } + } + } + + /// The kept snapshot, or `None` when no file exists. + fn read_kept(&self) -> crate::Result> { let path = self.dir.join(SNAPSHOT_FILE); let bytes = match std::fs::read(&path) { Ok(bytes) => bytes, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(e) => return Err(storage_error(&path, "read", e)), }; - let snapshot = SequencerSnapshot::decode(&bytes).map_err(codec_error)?; - let Ok((_, boundary, _)) = multi_raft.snapshot_metadata(SEQUENCER_GROUP_ID) else { - return Ok(false); - }; - if snapshot.applied_index() < boundary { - return Ok(false); - } - self.state_machine - .lock() - .unwrap_or_else(|p| p.into_inner()) - .restore_snapshot(snapshot); - Ok(true) + SequencerSnapshot::decode(&bytes) + .map(Some) + .map_err(codec_error) } } diff --git a/nodedb/src/control/cluster/start_raft/hooks.rs b/nodedb/src/control/cluster/start_raft/hooks.rs index e838d80d8..1033f62a2 100644 --- a/nodedb/src/control/cluster/start_raft/hooks.rs +++ b/nodedb/src/control/cluster/start_raft/hooks.rs @@ -54,7 +54,8 @@ pub(super) async fn build_hooks( // The sequencer group's snapshot: its state machine, captured on the // send path and kept durably on the receive path. A node whose sequencer // log starts right after an installed snapshot restores the state - // machine from it before any entry applies. + // machine from it before any entry applies. A log that starts above its + // first entry with no usable snapshot marks the history unknown. let sequencer_snapshots = Arc::new( crate::control::cluster::sequencer_snapshot::SequencerSnapshotStore::new( Arc::clone(sequencer_state_machine), diff --git a/nodedb/src/control/cluster/start_raft/loop_build.rs b/nodedb/src/control/cluster/start_raft/loop_build.rs index 2c53e7195..498d0004f 100644 --- a/nodedb/src/control/cluster/start_raft/loop_build.rs +++ b/nodedb/src/control/cluster/start_raft/loop_build.rs @@ -185,6 +185,8 @@ pub(super) async fn build_raft_loop( .map_err(|e| crate::Error::Config { detail: e.to_string(), })?; + // Each scheduler runs on the epoch duration of the sequencer feeding it. + let scheduler_config = SchedulerConfig::from_tuning(&shared.tuning.calvin, &sequencer_config); let (sequencer_inbox, sequencer_inbox_rx) = new_inbox(10_000, &sequencer_config); let (reservation_inbox, reservation_inbox_rx) = new_reservation_inbox(10_000); let ollp_orchestrator = Arc::new(OllpOrchestrator::new(OllpConfig::default())); @@ -208,7 +210,6 @@ pub(super) async fn build_raft_loop( ); let sequencer_metrics = Arc::clone(&sequencer_service.metrics); - let scheduler_config = SchedulerConfig::default(); spawn_vshard_schedulers(SpawnVshardSchedulersParams { handle, shared, diff --git a/nodedb/src/control/planner/calvin/abort_error.rs b/nodedb/src/control/planner/calvin/abort_error.rs index 6eef969e8..9ff3e12ec 100644 --- a/nodedb/src/control/planner/calvin/abort_error.rs +++ b/nodedb/src/control/planner/calvin/abort_error.rs @@ -13,6 +13,14 @@ use crate::Error; pub fn calvin_abort_error(reason: AbortReason) -> Error { match reason { AbortReason::ParticipantError => Error::CalvinParticipantError, + // A participant rejected the plans: they do not decode or route, or + // hold no work for it. A resubmit of the same plans fails the same + // way, so this is an internal error (`XX000`), never a retry class. + AbortReason::PlanRejected => Error::Internal { + detail: "cross-shard transaction aborted: a participant vShard rejected the \ + transaction's plans" + .to_owned(), + }, // Planned against a collection a purge and a same-name create replaced. AbortReason::CollectionSuperseded => Error::RetryableSchemaChanged { descriptor: "a collection the transaction writes was dropped and recreated".into(), @@ -39,6 +47,14 @@ mod tests { )); } + #[test] + fn a_rejected_plan_is_an_internal_error() { + assert!(matches!( + calvin_abort_error(AbortReason::PlanRejected), + Error::Internal { .. } + )); + } + #[test] fn a_superseded_collection_asks_for_a_retry() { assert!(matches!( diff --git a/nodedb/src/control/planner/calvin/dependent_recon.rs b/nodedb/src/control/planner/calvin/dependent_recon.rs index 27fb732fd..ca77a2230 100644 --- a/nodedb/src/control/planner/calvin/dependent_recon.rs +++ b/nodedb/src/control/planner/calvin/dependent_recon.rs @@ -134,7 +134,7 @@ pub fn is_edge_recon_plan(plan: &nodedb_physical::physical_plan::PhysicalPlan) - /// `submit_calvin_routed_assign`) that mirrors the doc write together with /// the derived EdgeDelete/EdgePut tasks, ATOMICALLY. A delete adds one /// `NodeEdgeGuard` per node and one `EdgeDelete` per incident edge. -/// 4. On a POST-EXEC predicate-drift mismatch or a guard's drift abort, +/// 4. On a `PredictionDrift` abort verdict, from a participant or a guard, /// re-scans (FRESH reconnaissance) and resubmits, via /// [`run_dependent_with_retry`]. /// @@ -213,7 +213,7 @@ async fn dispatch_dependent_edge_recon_inner( // OLLP path: the coordinator owns the retry loop. `run_dependent_with_retry` // submits + awaits the assignment/completion via the local registry and, on - // a post-exec predicate-drift mismatch, runs a FRESH pre-execution scan + // a `PredictionDrift` abort verdict, runs a FRESH pre-execution scan // (`rescan`) before resubmitting with the fresh prediction. let dep_task = tasks .iter() @@ -406,7 +406,7 @@ async fn dispatch_dependent_edge_recon_inner( } }; - // `rescan`: FRESH reconnaissance on each post-exec mismatch or drift. + // `rescan`: FRESH reconnaissance on each drift abort. let rescan = || { reconnoitre( state, diff --git a/nodedb/src/control/planner/calvin/retry_loop.rs b/nodedb/src/control/planner/calvin/retry_loop.rs index 0bea546d1..fbe58d25c 100644 --- a/nodedb/src/control/planner/calvin/retry_loop.rs +++ b/nodedb/src/control/planner/calvin/retry_loop.rs @@ -4,10 +4,10 @@ //! //! The retry loop for dependent-read (OLLP) Calvin transactions is owned by the //! coordinator (the pgwire handler), not the per-vshard scheduler. On a -//! post-exec predicate-drift mismatch the loop runs a FRESH pre-execution +//! `PredictionDrift` abort verdict the loop runs a FRESH pre-execution //! reconnaissance before resubmitting — a stale prediction can never converge -//! under predicate drift. The scheduler's only job on mismatch is to release the -//! aborted attempt's locks and signal the completion registry so this loop wakes. +//! under predicate drift. A participant that finds drift votes abort, drops +//! its slice at the verdict, and acks, which wakes this loop. use nodedb_cluster::calvin::{AbortReason, AttemptOutcome, CalvinCompletionRegistry, TxnId}; @@ -53,9 +53,9 @@ pub enum DependentOutcome { /// per attempt. /// /// This is the single owner of the submit → await-assignment → await-completion -/// → (mismatch ? re-scan : done) loop for dependent-read Calvin transactions. -/// On a POST-EXEC predicate-drift mismatch (the executor released the aborted -/// attempt's locks and the scheduler signalled the registry), the loop runs the +/// → (drift ? re-scan : done) loop for dependent-read Calvin transactions. +/// On a `PredictionDrift` abort verdict (every participant dropped the attempt +/// and released its locks before it acked), the loop runs the /// injected `rescan` closure to produce a FRESH prediction and resubmits — a /// stale prediction can never converge under predicate drift. On a PRE-ADMISSION /// failure (`OllpError` from the circuit-breaker / sequencer / tenant budget), @@ -156,31 +156,21 @@ where // for a reason a fresh reconnaissance cannot change, so surface it // to the client immediately instead of burning retries. The // verdict's reason picks the error: a stale read-set is SQLSTATE - // 40001, a participant error is not. `PredictionDrift` and - // `PartsLost` retry, in the arm below: a multi-part transaction - // whose parts a leader change lost staged nothing anywhere. + // 40001, a participant error is not, and rejected plans are an + // internal error. `PredictionDrift` and `PartsLost` retry, in the + // arm below: a multi-part transaction whose parts a leader change + // lost staged nothing anywhere. AttemptOutcome::Aborted { reason } if reason != AbortReason::PredictionDrift && reason != AbortReason::PartsLost => { return Err(calvin_abort_error(reason)); } - // Terminal, NON-retryable: the scheduler rejected the transaction's - // local plan routing and broadcast `TxnRoutingFailed`. A fresh - // reconnaissance can never fix a routing rejection, so surface it - // to the caller immediately instead of burning retries. - AttemptOutcome::Failed { detail } => { - return Err(Error::Internal { - detail: format!("calvin transaction routing failed: {detail}"), - }); - } - AttemptOutcome::Mismatch | AttemptOutcome::Aborted { .. } => { - // POST-EXEC predicate drift: an OLLP mismatch, or an abort - // verdict because a participant found state other than the - // reconnaissance predicted and wrote nothing. The scheduler - // already released the aborted attempt's locks before - // signalling the registry, so a FRESH reconnaissance is safe — - // and necessary, since the stale prediction can never converge - // under drift. + AttemptOutcome::Aborted { .. } => { + // A participant found state other than the reconnaissance + // predicted and wrote nothing, or the parts were lost. Every + // participant released the attempt's locks before it acked, + // so a FRESH reconnaissance is safe — and necessary, since the + // stale prediction can never converge under drift. if retry >= ollp_max_retries { return Err(Error::OllpExhausted { retries: ollp_max_retries.min(u8::MAX as u32) as u8, @@ -206,18 +196,18 @@ mod tests { //! returns a `RoutedAssignment` carrying a deterministic `(epoch, position)` — //! exactly the `(epoch, position)` the loop feeds to //! `register_completion_report(TxnId::new(epoch, position))`. The closure also forwards - //! that same `TxnId` to the fake over an mpsc channel; the fake then either calls - //! `note_ollp_mismatch(txn)` (first K submissions → `Mismatch`) or - //! `note_completion_ack(txn, 1)` (submission K+1, 1 participant → fires - //! `Completed`). The `rescan` closure increments a counter and returns a fresh - //! prediction vec. + //! that same `TxnId` to the fake over an mpsc channel; the fake stores a + //! verdict, `Abort(PredictionDrift)` for the first K submissions and + //! `Commit` after, then calls `note_completion_ack(txn, 1)` (1 participant → + //! fires the outcome). The `rescan` closure increments a counter and returns a + //! fresh prediction vec. //! //! The routed `submit` returns the assignment itself (the leader //! assigns and replies with `(epoch, position)`), so the loop never calls //! `register_submission` and the fake needs no `note_assigned`. The //! `(epoch, position)` source is deterministic on the test side: `epoch = //! inbox_seq`, `position = 0`. Both the `RoutedAssignment` returned by `submit` - //! AND the fake's `note_completion_ack` / `note_ollp_mismatch` use that same + //! AND the fake's `note_verdict` / `note_completion_ack` use that same //! `TxnId`, so they always match. //! //! Determinism: a current-thread runtime plus a bounded fake channel with enough @@ -230,6 +220,7 @@ mod tests { use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use crate::control::cluster::calvin::executor::ollp::config::OllpConfig; + use nodedb_cluster::calvin::VerdictOutcome; use nodedb_cluster::calvin::sequencer::error::SequencerError; /// Build an orchestrator with zero backoff so the retry loop runs instantly. @@ -242,24 +233,26 @@ mod tests { } /// Spawn the fake scheduler. It reads `TxnId` events (the same `TxnId` the loop - /// registers for completion) and, for the first `mismatch_count` events, signals - /// an OLLP mismatch; on the next event it acks completion with a single - /// participant (which fires `Completed`). The loop never calls - /// `register_submission`, so `note_assigned` is not needed. + /// registers for completion). For the first `drift_count` events it stores an + /// `Abort(PredictionDrift)` verdict, and a `Commit` verdict after. Each event + /// then acks with a single participant, which fires the outcome. The loop + /// never calls `register_submission`, so `note_assigned` is not needed. fn spawn_fake_scheduler( registry: Arc, mut rx: tokio::sync::mpsc::Receiver, - mismatch_count: u32, + drift_count: u32, ) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { let mut seen: u32 = 0; while let Some(txn) = rx.recv().await { - if seen < mismatch_count { - registry.note_ollp_mismatch(txn); + let verdict = if seen < drift_count { + VerdictOutcome::Abort(AbortReason::PredictionDrift) } else { - // 1 participant so a single ack completes the attempt. - registry.note_completion_ack(txn, 1); - } + VerdictOutcome::Commit + }; + registry.note_verdict(txn, verdict); + // 1 participant so a single ack completes the attempt. + registry.note_completion_ack(txn, 1); seen += 1; } }) @@ -278,7 +271,7 @@ mod tests { } #[test] - fn converges_after_two_mismatches() { + fn converges_after_two_drift_aborts() { let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -337,12 +330,12 @@ mod tests { assert_eq!( submit_calls.load(Ordering::SeqCst), 3, - "two mismatches + one success → three submits" + "two drift aborts + one success → three submits" ); assert_eq!( rescan_calls.load(Ordering::SeqCst), 2, - "fresh re-scan runs once per mismatch" + "fresh re-scan runs once per drift abort" ); drop(tx); let _ = fake.await; @@ -350,7 +343,7 @@ mod tests { } #[test] - fn exhausts_on_persistent_mismatch() { + fn exhausts_on_persistent_drift() { let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -359,7 +352,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let orchestrator = zero_backoff_orchestrator(); - // Mismatch on every attempt (large count → never acks). + // Drift on every attempt (large count → never commits). let (tx, rx) = tokio::sync::mpsc::channel::(16); let fake = spawn_fake_scheduler(Arc::clone(®istry), rx, u32::MAX); @@ -425,8 +418,8 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let orchestrator = zero_backoff_orchestrator(); - // No mismatch path needed: the first two submits fail pre-admission, the - // third admits and the fake immediately acks completion. + // No drift path needed: the first two submits fail pre-admission, the + // third admits and the fake immediately commits it. let (tx, rx) = tokio::sync::mpsc::channel::(16); let fake = spawn_fake_scheduler(Arc::clone(®istry), rx, 0); @@ -652,13 +645,13 @@ mod tests { tokio::spawn(async move { let mut first = true; while let Some(txn) = rx.recv().await { - if first { - registry.note_verdict( - txn, - nodedb_cluster::calvin::VerdictOutcome::Abort(reason), - ); - first = false; - } + let verdict = if first { + VerdictOutcome::Abort(reason) + } else { + VerdictOutcome::Commit + }; + first = false; + registry.note_verdict(txn, verdict); registry.note_completion_ack(txn, 1); } }) @@ -721,6 +714,17 @@ mod tests { assert_eq!(rescans, 1, "the drift abort reads again once"); } + #[test] + fn a_plan_rejected_abort_is_terminal() { + let (result, submits, rescans) = run_with_first_abort(AbortReason::PlanRejected); + assert!( + matches!(result, Err(Error::Internal { .. })), + "expected an internal error, got {result:?}" + ); + assert_eq!(submits, 1); + assert_eq!(rescans, 0); + } + #[test] fn a_participant_error_abort_is_terminal() { let (result, submits, rescans) = run_with_first_abort(AbortReason::ParticipantError); diff --git a/nodedb/src/control/planner/calvin/submit/local.rs b/nodedb/src/control/planner/calvin/submit/local.rs index 27574e6d3..1916773cf 100644 --- a/nodedb/src/control/planner/calvin/submit/local.rs +++ b/nodedb/src/control/planner/calvin/submit/local.rs @@ -236,34 +236,11 @@ pub(crate) async fn submit_prepared_and_await( .map_err(|_| Error::Internal { detail: "Calvin completion channel closed".to_owned(), })?; - let outcome = report.outcome; - // Terminal, NON-retryable: the scheduler rejected the transaction's local - // plan routing and broadcast `TxnRoutingFailed`. Surface it immediately — - // falling through to the RETURNING-drain below will silently report - // `Ok(None)` for a transaction that never applied. - if let AttemptOutcome::Failed { detail } = &outcome { - return Err(Error::Internal { - detail: format!("calvin transaction routing failed: {detail}"), - }); - } - // Terminal, NON-retryable: the global cross-shard verdict was ABORT and the - // writes were dropped. This is a fall-through chain, NOT a match — without - // this explicit check `Aborted` will fall through to the RETURNING drain - // below and silently return `Ok(None)`, reporting COMMIT SUCCESS for a - // transaction that never applied. The verdict's reason picks the error the - // client retries on. - if let AttemptOutcome::Aborted { reason } = &outcome { - return Err(calvin_abort_error(*reason)); - } - // The static (non-dependent) Calvin path never produces an OLLP mismatch — - // `note_ollp_mismatch` only fires on the dependent-predicate retry path — so - // this branch is unreachable at runtime today. It is kept as a typed error - // (never a panic) so any future mismatch signal on this channel surfaces - // deterministically instead of crashing. - if outcome == AttemptOutcome::Mismatch { - return Err(Error::Internal { - detail: "OLLP mismatch outcome on non-dependent Calvin path".to_owned(), - }); + // The global cross-shard verdict was ABORT and the writes were dropped. + // The verdict's reason picks the error the client sees. + match report.outcome { + AttemptOutcome::Completed => {} + AttemptOutcome::Aborted { reason } => return Err(calvin_abort_error(reason)), } if binds_authorization { crate::control::security::auth_lease::calvin_write_barrier(state).await?; diff --git a/nodedb/src/control/server/pgwire/handler/transaction_cmds/errors.rs b/nodedb/src/control/server/pgwire/handler/transaction_cmds/errors.rs index 54f4d9a4d..f21bba9b3 100644 --- a/nodedb/src/control/server/pgwire/handler/transaction_cmds/errors.rs +++ b/nodedb/src/control/server/pgwire/handler/transaction_cmds/errors.rs @@ -56,24 +56,4 @@ mod tests { other => panic!("expected PgWireError::UserError, got {other:?}"), } } - - #[test] - fn ollp_mismatch_is_not_57014() { - // Verify the OLLP mismatch arm (a genuine invariant violation) stays - // XX000 — a distinct code path from coordinator deadline cancellation. - let err = PgWireError::UserError(Box::new(ErrorInfo::new( - "ERROR".to_owned(), - "XX000".to_owned(), - "OLLP mismatch outcome on non-dependent Calvin path".to_owned(), - ))); - match err { - PgWireError::UserError(info) => { - assert_eq!( - info.code, "XX000", - "OLLP mismatch must stay XX000 (internal_error)" - ); - } - other => panic!("unexpected error variant: {other:?}"), - } - } } From cb473f06ac2f98e00769abbd3e9ad608a22d9303 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Fri, 9 Oct 2026 13:00:47 +0800 Subject: [PATCH 02/19] feat(calvin): add moded collection and unique locks and a typed stage vote The Calvin lock manager now takes moded requests. Row keys lock exclusive or shared, and a collection key carries intent, shared, or exclusive modes so a collection-wide writer or predicate reader orders against row writers of the same collection. UNIQUE values a write claims lock as their own keys, so transactions that claim one value serialize. The planner derives these keys, and EngineKeySet and LockKeyWire gain Collection and Unique variants. A Calvin stage response now carries a typed StageVote (commit, serialization conflict, prediction drift, participant error) in place of the optional read_set_valid flag. A stage response with no readable vote halts apply instead of counting as a commit. --- .../src/calvin/sequencer/validator.rs | 35 +- nodedb-cluster/src/calvin/types/lock_wire.rs | 16 + .../src/calvin/types/read_write_set.rs | 22 +- .../src/calvin/types/transaction.rs | 2 +- nodedb-types/src/calvin.rs | 38 +- nodedb/src/bridge/dispatch/drain.rs | 2 +- nodedb/src/bridge/dispatch/enqueue.rs | 4 +- nodedb/src/bridge/dispatch/response_poll.rs | 10 +- nodedb/src/bridge/envelope/mod.rs | 2 + nodedb/src/bridge/envelope/response.rs | 20 +- nodedb/src/bridge/envelope/stage_vote.rs | 81 +++ nodedb/src/control/array_catalog/ddl.rs | 2 +- .../control/array_sync/raft_apply/common.rs | 2 +- .../post_apply/async_dispatch/array.rs | 2 +- .../post_apply/async_dispatch/core_fanout.rs | 2 +- .../post_apply/async_dispatch/crdt_compact.rs | 2 +- .../control/cluster/array_executor/refusal.rs | 2 +- .../driver/core/commit_resolve/mod.rs | 2 +- .../driver/core/commit_resolve/verdict.rs | 7 +- .../driver/core/commit_resolve/vote.rs | 159 +++++- .../scheduler/driver/core/completion_route.rs | 6 +- .../driver/core/dispatch/active_dispatch.rs | 2 +- .../driver/core/dispatch/incarnation.rs | 7 +- .../driver/core/dispatch/static_dispatch.rs | 2 +- .../driver/core/dispatch/unstaged.rs | 7 +- .../calvin/scheduler/driver/core/halt.rs | 3 + .../calvin/scheduler/driver/core/intake.rs | 4 +- .../scheduler/driver/core/owed/retry.rs | 12 +- .../calvin/scheduler/driver/core/process.rs | 3 +- .../scheduler/driver/core/staged_vote.rs | 162 ++++-- .../scheduler/driver/core/test_support.rs | 8 +- .../calvin/scheduler/driver/helpers.rs | 126 +++-- .../cluster/calvin/scheduler/driver/types.rs | 9 +- .../calvin/scheduler/lock/lock_entry.rs | 92 +++- .../cluster/calvin/scheduler/lock/lock_key.rs | 11 + .../calvin/scheduler/lock/manager/acquire.rs | 521 +++++++++--------- .../calvin/scheduler/lock/manager/classify.rs | 100 ++++ .../scheduler/lock/manager/introspection.rs | 22 +- .../calvin/scheduler/lock/manager/mod.rs | 20 +- .../calvin/scheduler/lock/manager/release.rs | 174 +++--- .../scheduler/lock/manager/try_acquire.rs | 62 ++- .../calvin/scheduler/lock/manager/types.rs | 68 +-- .../scheduler/lock/manager/wound_wait.rs | 367 ++++++------ .../cluster/calvin/scheduler/lock/reap.rs | 74 ++- .../cluster/calvin/scheduler/metrics.rs | 2 + nodedb/src/control/crdt_admission.rs | 2 +- .../error_map/system_dispatch_refusal.rs | 2 +- nodedb/src/control/gateway/outcome.rs | 2 +- .../src/control/insert_select/orchestrator.rs | 2 +- nodedb/src/control/local_dispatch/collect.rs | 8 +- .../control/local_dispatch/error_status.rs | 2 +- nodedb/src/control/orchestrated_write.rs | 4 +- .../control/planner/calvin/submit/assign.rs | 2 + .../control/planner/calvin/submit/local.rs | 3 +- .../src/control/planner/calvin/submit/mod.rs | 1 + .../control/planner/calvin/submit/routed.rs | 1 + .../planner/calvin/submit/unique_claims.rs | 517 +++++++++++++++++ .../calvin/tx_class/dependent_builder.rs | 113 +++- .../control/planner/calvin/tx_class/shared.rs | 313 ++++++----- .../planner/calvin/tx_class/static_builder.rs | 82 +++ .../calvin/tx_class/write_keys/columnar.rs | 4 +- .../calvin/tx_class/write_keys/document.rs | 16 +- .../calvin/tx_class/write_keys/graph.rs | 8 +- .../planner/calvin/tx_class/write_keys/kv.rs | 8 +- .../planner/calvin/tx_class/write_keys/mod.rs | 2 +- .../calvin/tx_class/write_keys/plan.rs | 59 +- .../planner/calvin/tx_class/write_keys/set.rs | 23 +- .../calvin/tx_class/write_keys/vector.rs | 4 +- .../procedural/executor/core/dispatch.rs | 2 +- nodedb/src/control/request_tracker.rs | 4 +- nodedb/src/control/server/broadcast.rs | 4 +- .../control/server/dispatch_utils/dispatch.rs | 4 +- .../server/dispatch_utils/durable_write.rs | 2 +- .../server/dispatch_utils/minted/owned.rs | 2 +- .../server/dispatch_utils/minted/resolve.rs | 2 +- .../server/dispatch_utils/owner_read.rs | 2 +- .../exchange/all_cores/base_snapshot.rs | 2 +- .../control/server/exchange/core_outcome.rs | 2 +- .../src/control/server/exchange/response.rs | 2 +- .../control/server/graph_dispatch/helpers.rs | 2 +- .../server/native/dispatch/raw_dispatch.rs | 2 +- .../server/native/dispatch/single_task.rs | 2 +- .../pgwire/handler/dispatch/replicated.rs | 2 +- .../handler/routing/dispatch_loop/run.rs | 2 +- nodedb/src/control/server/result_stream.rs | 8 +- .../server/shared/clone_read/dispatch.rs | 2 +- .../control/server/shared/clone_write/util.rs | 2 +- .../ddl/neutral/kv_sorted_index/dispatch.rs | 2 +- .../server/shared/ddl/neutral/rate_gate.rs | 2 +- .../ddl/neutral/version_history/restore.rs | 2 +- .../server/shared/ddl/user_dispatch.rs | 2 +- nodedb/src/control/server/shared/metering.rs | 2 +- .../control/server/shared/response_payload.rs | 2 +- .../server/shared/session/lifecycle.rs | 2 +- .../control/server/shared/session/read_set.rs | 12 + .../server/shared/session/savepoint_ops.rs | 2 +- .../server/shared/write_admission/gate.rs | 11 +- .../server/shared/write_admission/route.rs | 2 +- .../server/sync/raft_dispatch/response.rs | 2 +- nodedb/src/control/shutdown/data_plane.rs | 2 +- nodedb/src/control/state/calvin_ack_result.rs | 2 +- nodedb/src/control/state/test_core.rs | 2 +- nodedb/src/control/system_txn/tests.rs | 2 +- nodedb/src/control/write_resolve/propose.rs | 2 +- .../src/data/executor/core_loop/response.rs | 8 +- nodedb/src/data/executor/core_loop/tick.rs | 2 +- .../handlers/columnar_write/insert.rs | 2 +- .../handlers/control/calvin/active_passive.rs | 45 +- .../handlers/control/calvin/shared.rs | 68 ++- .../handlers/control/calvin/static_stage.rs | 111 +++- .../handlers/control/calvin_passive_read.rs | 3 + .../handlers/query_collection_size.rs | 2 +- .../executor/handlers/timeseries/ingest.rs | 2 +- .../handlers/timeseries/ingest_dispatch.rs | 2 +- .../handlers/timeseries/ingest_resolved.rs | 2 +- .../data/executor/handlers/timeseries/scan.rs | 2 +- nodedb/src/data/runtime/event_loop.rs | 2 +- .../inproc/cases/calvin_two_phase_apply.rs | 34 +- .../cases/request_tracker_backpressure.rs | 2 +- .../tests/inproc/cases/startup_gate_pgwire.rs | 2 +- .../inproc/cases/write_admission_fence.rs | 8 +- 121 files changed, 2732 insertions(+), 1122 deletions(-) create mode 100644 nodedb/src/bridge/envelope/stage_vote.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/lock/manager/classify.rs create mode 100644 nodedb/src/control/planner/calvin/submit/unique_claims.rs diff --git a/nodedb-cluster/src/calvin/sequencer/validator.rs b/nodedb-cluster/src/calvin/sequencer/validator.rs index b1c9166c7..d2e45f60f 100644 --- a/nodedb-cluster/src/calvin/sequencer/validator.rs +++ b/nodedb-cluster/src/calvin/sequencer/validator.rs @@ -19,8 +19,8 @@ //! //! # Conflict handling (read-aware) //! -//! The commit lockset is all-exclusive over each txn's read ∪ write keys, so -//! same-key transactions serialize by `position`. Per-key last-write-LSN +//! The scheduler locks a txn's write keys exclusive and its read keys shared, +//! so a reader and a writer of one key serialize by `position`. Per-key last-write-LSN //! validation means a reader `R` of key `K` aborts at commit only if a //! *lower-position* writer `W` of `K` committed first (its write LSN exceeds //! `R`'s read version). Giving `R` a lower position than `W` converts that @@ -63,7 +63,8 @@ type KeyId = (u8, String, Vec); /// A single flattened key drawn from one transaction's read or write set. #[derive(Debug)] struct FlatKey { - /// Discriminant tag for the engine variant (Document=0, Vector=1, Kv=2, Edge=3). + /// Discriminant tag for the key-set variant (Document=0, Vector=1, Kv=2, + /// Edge=3, Array=4, Collection=5, Unique=6). discriminant: u8, /// Static engine name used when building conflict-context keys. engine_name: &'static str, @@ -153,6 +154,34 @@ fn flatten_set(set: &ReadWriteSet) -> Vec { }); } } + // The whole collection: one key with no row bytes. + EngineKeySet::Collection { collection, .. } => { + out.push(FlatKey { + discriminant: 5, + engine_name: "collection", + collection: collection.clone(), + key_bytes: Vec::new(), + }); + } + // One key per claimed value: the index name, a zero byte, the value. + EngineKeySet::Unique { + collection, + index, + values, + } => { + for value in values.as_slice() { + let mut key_bytes = Vec::with_capacity(index.len() + 1 + value.len()); + key_bytes.extend_from_slice(index.as_bytes()); + key_bytes.push(0); + key_bytes.extend_from_slice(value); + out.push(FlatKey { + discriminant: 6, + engine_name: "unique", + collection: collection.clone(), + key_bytes, + }); + } + } } } out diff --git a/nodedb-cluster/src/calvin/types/lock_wire.rs b/nodedb-cluster/src/calvin/types/lock_wire.rs index ab5191ab1..ca16cd323 100644 --- a/nodedb-cluster/src/calvin/types/lock_wire.rs +++ b/nodedb-cluster/src/calvin/types/lock_wire.rs @@ -57,6 +57,14 @@ pub enum LockKeyWire { src: u32, dst: u32, }, + /// A whole collection. + Collection { collection: String }, + /// One claimed value of one UNIQUE index. + Unique { + collection: String, + index: String, + value: Vec, + }, } // ── ReleaseReason ───────────────────────────────────────────────────────────── @@ -116,6 +124,14 @@ mod tests { src: 1, dst: 2, }, + LockKeyWire::Collection { + collection: "orders".to_owned(), + }, + LockKeyWire::Unique { + collection: "users".to_owned(), + index: "users_email".to_owned(), + value: b"a@x".to_vec(), + }, ]; for v in variants { let bytes = zerompk::to_msgpack_vec(&v).expect("encode"); diff --git a/nodedb-cluster/src/calvin/types/read_write_set.rs b/nodedb-cluster/src/calvin/types/read_write_set.rs index ea51c97a1..55265e90d 100644 --- a/nodedb-cluster/src/calvin/types/read_write_set.rs +++ b/nodedb-cluster/src/calvin/types/read_write_set.rs @@ -38,10 +38,12 @@ impl ReadWriteSet { /// Derive the set of vShards participating in this read/write set. /// - /// For Document/Vector/KV entries the vshard is derived from the - /// collection name (collection-level routing, consistent with the - /// per-vshard Raft groups that own each collection). KV collections - /// are also assigned a single vshard at creation time. + /// For Document/Vector/KV/Unique entries, and a Collection entry that + /// names no vShards, the vshard is derived from the collection name + /// (collection-level routing, consistent with the per-vshard Raft groups + /// that own each collection). KV collections are also assigned a single + /// vshard at creation time. A Collection entry that names vShards + /// participates on exactly those. /// /// For Edge entries the participating vShards are the edge's /// `home_vshards` (the `from_key(src)` / `from_key(dst)` key-hashed @@ -81,9 +83,19 @@ impl ReadWriteSet { } } } + EngineKeySet::Collection { vshards: homes, .. } if !homes.is_empty() => { + for &home in homes.as_slice() { + let vshard = VShardId::new(home); + if seen.insert(vshard.as_u32()) { + result.push(vshard); + } + } + } EngineKeySet::Document { .. } | EngineKeySet::Vector { .. } - | EngineKeySet::Kv { .. } => { + | EngineKeySet::Kv { .. } + | EngineKeySet::Collection { .. } + | EngineKeySet::Unique { .. } => { let vshard = CollectionKey::from_qualified_str(database_id, engine_set.collection())? .vshard(); diff --git a/nodedb-cluster/src/calvin/types/transaction.rs b/nodedb-cluster/src/calvin/types/transaction.rs index b53080eb5..0eba9948e 100644 --- a/nodedb-cluster/src/calvin/types/transaction.rs +++ b/nodedb-cluster/src/calvin/types/transaction.rs @@ -75,7 +75,7 @@ pub struct TxClass { /// Each entry carries the responding shard's write-LSN watermark at read /// time plus the point/predicate identity, so a participant can validate /// the read at the commit serialization point (the local commit vote on - /// `read_set_valid`). Empty for pure-write and autocommit transactions. + /// a stage response). Empty for pure-write and autocommit transactions. #[serde(default)] #[msgpack(default)] pub versioned_reads: VersionedReadSet, diff --git a/nodedb-types/src/calvin.rs b/nodedb-types/src/calvin.rs index 003662c56..c7b9c9208 100644 --- a/nodedb-types/src/calvin.rs +++ b/nodedb-types/src/calvin.rs @@ -121,6 +121,27 @@ pub enum EngineKeySet { collection: String, vshards: SortedVec, }, + /// A whole collection on each vShard in `vshards`, or on the + /// collection's own vShard when `vshards` is empty. + /// + /// In a write set the transaction locks the collection exclusively: a + /// truncate, bulk, predicate, or identity-less write. In a read set it + /// locks the collection shared: a predicate, scan, or index read. A read + /// that observed the collection on named vShards, such as a cross-shard + /// graph read, names them in `vshards`. + Collection { + collection: String, + vshards: SortedVec, + }, + /// UNIQUE values a write claims in one index of `collection`. + /// + /// Each value is the canonical index value the Data Plane judges. Two + /// transactions that claim one value lock one key, so they serialize. + Unique { + collection: String, + index: String, + values: SortedVec>, + }, } impl EngineKeySet { @@ -142,6 +163,15 @@ impl EngineKeySet { Self::Edge { edges, .. } => edges.len() * 8, // Array: one u32 vShard each. Self::Array { vshards, .. } => vshards.len() * 4, + // Collection: the name plus one u32 vShard each. + Self::Collection { + collection, + vshards, + } => collection.len() + vshards.len() * 4, + // Unique: the index name plus each value's bytes. + Self::Unique { index, values, .. } => { + index.len() + values.iter().map(|v| v.len()).sum::() + } } } @@ -152,7 +182,9 @@ impl EngineKeySet { | Self::Vector { collection, .. } | Self::Kv { collection, .. } | Self::Edge { collection, .. } - | Self::Array { collection, .. } => collection, + | Self::Array { collection, .. } + | Self::Collection { collection, .. } + | Self::Unique { collection, .. } => collection, } } @@ -164,6 +196,8 @@ impl EngineKeySet { Self::Kv { keys, .. } => keys.is_empty(), Self::Edge { edges, .. } => edges.is_empty(), Self::Array { vshards, .. } => vshards.is_empty(), + Self::Collection { .. } => false, + Self::Unique { values, .. } => values.is_empty(), } } } @@ -309,7 +343,7 @@ pub struct VersionedReadEntry { /// accumulate no session read-set). Populated at commit time from the neutral /// session read-set, and validated per-participant against the local /// write-version index at Calvin stage time (the commit vote on -/// `read_set_valid`). +/// a stage response). #[derive( Debug, Clone, diff --git a/nodedb/src/bridge/dispatch/drain.rs b/nodedb/src/bridge/dispatch/drain.rs index 79dba5960..22b9fdf0b 100644 --- a/nodedb/src/bridge/dispatch/drain.rs +++ b/nodedb/src/bridge/dispatch/drain.rs @@ -168,7 +168,7 @@ impl Dispatcher { shutdown drain deadline" ), })), - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }); diff --git a/nodedb/src/bridge/dispatch/enqueue.rs b/nodedb/src/bridge/dispatch/enqueue.rs index 29249e480..31ddff03f 100644 --- a/nodedb/src/bridge/dispatch/enqueue.rs +++ b/nodedb/src/bridge/dispatch/enqueue.rs @@ -322,7 +322,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }, @@ -387,7 +387,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), }, diff --git a/nodedb/src/bridge/dispatch/response_poll.rs b/nodedb/src/bridge/dispatch/response_poll.rs index 5eba45dbc..54673aa25 100644 --- a/nodedb/src/bridge/dispatch/response_poll.rs +++ b/nodedb/src/bridge/dispatch/response_poll.rs @@ -93,7 +93,7 @@ impl Dispatcher { "core-{core_id} is gone; the request can never be executed" ), })), - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }); @@ -133,7 +133,7 @@ mod tests { payload: Payload::from_vec(b"result".to_vec()), watermark_lsn: Lsn::new(42), error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), }, @@ -164,7 +164,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }, @@ -293,7 +293,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), }, @@ -388,7 +388,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }, diff --git a/nodedb/src/bridge/envelope/mod.rs b/nodedb/src/bridge/envelope/mod.rs index 371b131b2..337b116e2 100644 --- a/nodedb/src/bridge/envelope/mod.rs +++ b/nodedb/src/bridge/envelope/mod.rs @@ -8,6 +8,7 @@ pub mod error_code_from; pub mod payload; pub mod request; pub mod response; +pub mod stage_vote; pub mod status; pub mod sync_hold; @@ -17,5 +18,6 @@ pub use nodedb_physical::physical_plan::PhysicalPlan; pub use payload::Payload; pub use request::{Admission, ExemptReason, Request}; pub use response::{EdgeImage, Response, RowEffect, RowVersion, WriteSetEntry}; +pub use stage_vote::StageVote; pub use status::{Priority, Status}; pub use sync_hold::SyncHold; diff --git a/nodedb/src/bridge/envelope/response.rs b/nodedb/src/bridge/envelope/response.rs index 53fff8144..378d90f2c 100644 --- a/nodedb/src/bridge/envelope/response.rs +++ b/nodedb/src/bridge/envelope/response.rs @@ -4,6 +4,7 @@ use super::error_code::ErrorCode; use super::payload::Payload; +use super::stage_vote::StageVote; use super::status::Status; use crate::types::{Lsn, RequestId}; use nodedb_types::RowIdentity; @@ -206,18 +207,11 @@ pub struct Response { /// Error code if status is not Ok. pub error_code: Option>, - /// Whether this response's originating transaction found its slice of the - /// versioned read-set still current against the local write versions. - /// `Some(true)` = still current (or no reads observed for this slice); - /// `Some(false)` = at least one read was superseded; `None` = the response - /// did not carry a read-set check (reads, control ops, and every - /// non-transaction response). + /// The local commit vote of a Calvin stage response. /// - /// For the direct-apply (dependent/active, fast-path) path this is reporting - /// only — the apply commits regardless. For a staged static Calvin - /// transaction it is the LOCAL COMMIT VOTE: the scheduler flushes the staged - /// buffer to base on `Some(true)` and drops it on `Some(false)`. - pub read_set_valid: Option, + /// Every Calvin stage path sets it. `None` means the response is not a + /// Calvin stage outcome. It never means an unknown vote. + pub stage_vote: Option, /// Row-level effects the Control Plane turns into durable redo records /// *after* the Data Plane applied them (see [`WriteSetEntry`]). Empty for @@ -239,7 +233,7 @@ mod tests { payload: Payload::from_vec(b"result".to_vec()), watermark_lsn: Lsn::new(42), error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }; @@ -258,7 +252,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: Some(Box::new(ErrorCode::DeadlineExceeded)), - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }; diff --git a/nodedb/src/bridge/envelope/stage_vote.rs b/nodedb/src/bridge/envelope/stage_vote.rs new file mode 100644 index 000000000..c62144995 --- /dev/null +++ b/nodedb/src/bridge/envelope/stage_vote.rs @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The local vote a Calvin stage response carries to its scheduler. + +use super::error_code::ErrorCode; + +/// A participant's local vote on its slice of a staged Calvin transaction. +/// +/// Every Calvin stage path sets one on its response. No other response +/// carries one. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StageVote { + /// The slice staged, and its slice of the read-set is still current. + Commit, + /// The slice staged, but a read it validates was superseded. + SerializationConflict, + /// A state the coordinator predicted moved. The slice holds no staged + /// state, and the coordinator reads again and retries. + PredictionDrift, + /// The stage failed. The slice holds no staged state. + ParticipantError, +} + +impl StageVote { + /// The vote of a stage that refused with `error`. + /// + /// `OllpRetryRequired` reports a moved prediction. Every other error is a + /// participant error. + pub fn of_stage_error(error: &ErrorCode) -> Self { + if matches!(error, ErrorCode::OllpRetryRequired) { + Self::PredictionDrift + } else { + Self::ParticipantError + } + } + + /// The vote of a staged slice whose read-set check found it `current`. + pub fn of_read_set(current: bool) -> Self { + if current { + Self::Commit + } else { + Self::SerializationConflict + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ollp_retry_refusal_votes_prediction_drift() { + assert_eq!( + StageVote::of_stage_error(&ErrorCode::OllpRetryRequired), + StageVote::PredictionDrift + ); + } + + #[test] + fn any_other_refusal_votes_participant_error() { + assert_eq!( + StageVote::of_stage_error(&ErrorCode::DeadlineExceeded), + StageVote::ParticipantError + ); + assert_eq!( + StageVote::of_stage_error(&ErrorCode::Internal { + detail: "stage".into() + }), + StageVote::ParticipantError + ); + } + + #[test] + fn read_set_check_maps_to_commit_or_serialization_conflict() { + assert_eq!(StageVote::of_read_set(true), StageVote::Commit); + assert_eq!( + StageVote::of_read_set(false), + StageVote::SerializationConflict + ); + } +} diff --git a/nodedb/src/control/array_catalog/ddl.rs b/nodedb/src/control/array_catalog/ddl.rs index fd3c8e08c..d17b60428 100644 --- a/nodedb/src/control/array_catalog/ddl.rs +++ b/nodedb/src/control/array_catalog/ddl.rs @@ -99,7 +99,7 @@ pub(crate) async fn run_trusted_array_ddl( payload: Payload::from_vec(payload), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }) diff --git a/nodedb/src/control/array_sync/raft_apply/common.rs b/nodedb/src/control/array_sync/raft_apply/common.rs index 4e9635db1..80db089b4 100644 --- a/nodedb/src/control/array_sync/raft_apply/common.rs +++ b/nodedb/src/control/array_sync/raft_apply/common.rs @@ -351,7 +351,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs index 447bfd416..58d4b9a83 100644 --- a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs +++ b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs @@ -210,7 +210,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }, diff --git a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/core_fanout.rs b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/core_fanout.rs index dfe5b6060..24e933ee2 100644 --- a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/core_fanout.rs +++ b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/core_fanout.rs @@ -224,7 +224,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/crdt_compact.rs b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/crdt_compact.rs index 748909ca3..87322a30d 100644 --- a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/crdt_compact.rs +++ b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/crdt_compact.rs @@ -188,7 +188,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code, - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }, diff --git a/nodedb/src/control/cluster/array_executor/refusal.rs b/nodedb/src/control/cluster/array_executor/refusal.rs index fc3f23ad5..44e68cd03 100644 --- a/nodedb/src/control/cluster/array_executor/refusal.rs +++ b/nodedb/src/control/cluster/array_executor/refusal.rs @@ -200,7 +200,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/mod.rs index 69168ac82..76ec99820 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/mod.rs @@ -4,7 +4,7 @@ //! //! A static Calvin dispatch STAGES its transaction on the Data Plane (validate //! the read-set + buffer the plans, no base mutation). Its executor response -//! carries the local commit vote on `read_set_valid`. This module drives the +//! carries the local commit vote on `stage_vote`. This module drives the //! final step: dispatch a flush (commit, after `commit_redo` has WAL-appended //! the resolved `TransactionRedo`) or drop (abort) of the staged buffer, wait //! for its response, then run the commit tail (deposit applied result, record diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs index 4b871743f..e784f0f8c 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs @@ -225,7 +225,7 @@ mod tests { use super::*; use crate::bridge::dispatch::CoreChannelDataSide; use crate::bridge::envelope::ErrorCode; - use crate::bridge::envelope::{Payload, Status}; + use crate::bridge::envelope::{Payload, StageVote, Status}; use crate::control::cluster::calvin::scheduler::driver::core::halt::HaltReason; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ await_data_plane_request, build_test_scheduler_with_data_side, error_response, @@ -434,7 +434,10 @@ mod tests { // Local staging votes only park their own staged slices; neither the // affirmative nor the failed participant can resolve or drop unilaterally. - first_scheduler.resolve_staged_commit(txn_id, &staged_response(Status::Ok, Some(true))); + first_scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); second_scheduler.resolve_staged_commit(txn_id, &staged_response(Status::Error, None)); for (scheduler, data_side) in [ (&first_scheduler, &mut first_data), diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs index f10a56bfe..d6824170e 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs @@ -6,7 +6,9 @@ use std::sync::atomic::Ordering; use std::time::Instant; use crate::bridge::envelope::Response; -use crate::control::cluster::calvin::scheduler::driver::core::halt::error_response_text; +use crate::control::cluster::calvin::scheduler::driver::core::halt::{ + HaltReason, HaltStep, error_response_text, +}; use crate::control::cluster::calvin::scheduler::driver::core::owed::SchedulerProposal; use crate::control::cluster::calvin::scheduler::driver::core::scheduler::Scheduler; use crate::control::cluster::calvin::scheduler::driver::core::staged_vote::{ @@ -20,12 +22,12 @@ impl Scheduler { /// PARK it on the cross-shard commit barrier awaiting the durable GLOBAL /// verdict — it does NOT self-decide flush-or-drop on its local vote. /// - /// The staged executor response is validate-only: its `read_set_valid` is - /// this shard's local commit vote (`Some(true)` => commit, `Some(false)` => - /// abort; a `None` from the active/dependent path is treated as commit). The - /// leader proposes that vote via the sequencer Raft group; the sequencer - /// aggregates all participants' votes into a single authoritative - /// `SequencerEntry::Verdict`, applied on every replica. + /// The staged executor response is validate-only: its `stage_vote` is this + /// shard's local commit vote, read by [`staged_commit_vote`]. A stage + /// response the scheduler cannot read a vote from halts apply. It never + /// counts as a commit. The leader proposes that vote via the sequencer + /// Raft group. The sequencer aggregates all participants' votes into a + /// single authoritative `SequencerEntry::Verdict`, applied on every replica. /// /// This method moves the txn to [`CommitState::AwaitingVerdict`] WITHOUT /// dispatching a resolve or drop, then immediately probes @@ -41,10 +43,9 @@ impl Scheduler { txn_id: TxnId, staged_response: &Response, ) { - // A staged error is always an abort vote. Only successful staged - // responses can use `None` for the dependent-read path; accepting an - // error-plus-None as commit will let a failed participant flush after - // its peers received a global commit verdict. + // A staged error is always an abort vote. A response with no readable + // vote halts: counting it as a commit will let a failed participant + // flush after its peers received a global commit verdict. let superseded = self .pending .get(&txn_id) @@ -55,10 +56,20 @@ impl Scheduler { match staged_commit_vote(staged_response) { // A read another node served is numbered in that node's WAL: // this node's versions cannot show it still current. - StagedVote::Commit if self.validates_read_served_elsewhere(txn_id) => { + Ok(StagedVote::Commit) if self.validates_read_served_elsewhere(txn_id) => { StagedVote::SerializationConflict } - vote => vote, + Ok(vote) => vote, + Err(error) => { + self.metrics.record_executor_error(); + self.halt_apply( + txn_id, + HaltReason::StageVoteInvalid, + HaltStep::Stage, + format!("{error}: {}", error_response_text("stage", staged_response)), + ); + return; + } } }; @@ -152,3 +163,125 @@ impl Scheduler { }) } } + +#[cfg(test)] +mod tests { + use nodedb_cluster::calvin::{AbortReason, SequencerEntry}; + + use super::*; + use crate::bridge::envelope::{StageVote, Status}; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ + CapturingProposer, elect_data_group_leader, + }; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler, make_sequenced_txn, staged_pending, staged_response, + }; + + const VSHARD: u32 = 7; + + /// A data-group leader holding `txn_id` staged, with a proposer that + /// accepts every proposal. + fn staged_leader( + txn_id: TxnId, + ) -> ( + Scheduler, + tempfile::TempDir, + std::sync::Arc, + ) { + let (mut scheduler, dir) = build_test_scheduler(VSHARD); + let proposer = CapturingProposer::accepting(); + scheduler.sequencer_proposer = proposer.clone(); + elect_data_group_leader(&scheduler); + scheduler.pending.insert( + txn_id, + staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id), + ); + (scheduler, dir, proposer) + } + + #[tokio::test] + async fn each_stage_vote_proposes_its_own_vote() { + let cases = [ + (Status::Ok, StageVote::Commit, None), + ( + Status::Ok, + StageVote::SerializationConflict, + Some(AbortReason::SerializationConflict), + ), + ( + Status::Error, + StageVote::PredictionDrift, + Some(AbortReason::PredictionDrift), + ), + ( + Status::Error, + StageVote::ParticipantError, + Some(AbortReason::ParticipantError), + ), + ]; + for (status, vote, abort) in cases { + let txn_id = TxnId::new(30, 1); + let (mut scheduler, _dir, proposer) = staged_leader(txn_id); + + scheduler.resolve_staged_commit(txn_id, &staged_response(status, Some(vote))); + + let expected = match abort { + None => SequencerEntry::Vote { + epoch: 30, + position: 1, + vshard: VSHARD, + }, + Some(reason) => SequencerEntry::AbortVote { + epoch: 30, + position: 1, + vshard: VSHARD, + reason, + }, + }; + assert_eq!(proposer.accepted(), vec![expected], "{vote:?}"); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingVerdict), + "{vote:?}" + ); + assert!(scheduler.apply_halt().is_none(), "{vote:?}"); + } + } + + #[tokio::test] + async fn a_successful_stage_response_without_a_vote_halts_and_never_votes() { + let txn_id = TxnId::new(31, 0); + let (mut scheduler, _dir, proposer) = staged_leader(txn_id); + + scheduler.resolve_staged_commit(txn_id, &staged_response(Status::Ok, None)); + + assert_eq!(proposer.attempt_count(), 0, "no vote is proposed"); + assert_eq!( + scheduler.apply_halt().map(|h| h.reason), + Some(HaltReason::StageVoteInvalid) + ); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Staged), + "the txn does not park on the verdict barrier" + ); + assert!(!scheduler.applied.is_applied(31, 0)); + } + + #[tokio::test] + async fn a_commit_vote_on_an_error_response_halts_and_never_votes() { + let txn_id = TxnId::new(32, 0); + let (mut scheduler, _dir, proposer) = staged_leader(txn_id); + + scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Error, Some(StageVote::Commit)), + ); + + assert_eq!(proposer.attempt_count(), 0, "no vote is proposed"); + assert_eq!( + scheduler.apply_halt().map(|h| h.reason), + Some(HaltReason::StageVoteInvalid) + ); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs index 99de7fa54..576e3fb9b 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs @@ -73,9 +73,9 @@ impl Scheduler { match commit_state { CommitState::Staged => { // Stage failures remain participants in the barrier: - // `resolve_staged_commit` turns any `Status::Error` into an - // explicit false vote, parks locally, and waits for the - // durable global verdict before issuing a drop. + // `resolve_staged_commit` proposes the response's abort vote, + // parks locally, and waits for the durable global verdict + // before it issues a drop. self.resolve_staged_commit(txn_id, &response); } CommitState::AwaitingRedoResolve => { diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs index 74329ac00..54caa0015 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs @@ -112,7 +112,7 @@ impl Scheduler { // buffer, no base apply); its response drives the same // resolve → redo → flush as the static path, for // WAL-only-restart durability. `resolve_staged_commit` reads the - // `read_set_valid: None` the active handler returns as "commit". + // `stage_vote` the active handler sets. commit_state: super::super::super::types::CommitState::Staged, // Set only once the txn parks in `AwaitingVerdict`. verdict_deadline: None, diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs index d23c6dc0a..170867c3d 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs @@ -170,7 +170,7 @@ mod tests { make_sequenced_txn, scheduler_with_pending, staged_response, }; use super::*; - use crate::bridge::envelope::Status; + use crate::bridge::envelope::{StageVote, Status}; use crate::control::cluster::calvin::scheduler::driver::types::CommitState; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; use crate::control::security::catalog::StoredCollection; @@ -241,7 +241,10 @@ mod tests { if let Some(pending) = scheduler.pending.get_mut(&txn_id) { pending.superseded = true; } - scheduler.resolve_staged_commit(txn_id, &staged_response(Status::Ok, Some(true))); + scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); let pending = scheduler.pending.get(&txn_id).expect("still parked"); assert_eq!(pending.commit_state, CommitState::AwaitingVerdict); assert!(pending.stage_error.is_some(), "a COMMIT verdict halts it"); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/static_dispatch.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/static_dispatch.rs index f3983108e..42b1a2b2b 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/static_dispatch.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/static_dispatch.rs @@ -234,7 +234,7 @@ impl Scheduler { /// the validate-only read path (`plans` empty). Both carry the txn's FULL /// `versioned_reads` to the apply core, which validates the LOCAL slice of /// the read-set — whether or not `plans` is empty — and returns the commit - /// vote on `read_set_valid`. A validate-only task has `has_primary_write == + /// vote on `stage_vote`. A validate-only task has `has_primary_write == /// false`, so it deposits no result sidecar entry, exactly as intended. /// /// The txn enters `pending` before the dispatch, so a stage refused at diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs index e7c0bea10..26a551366 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs @@ -115,7 +115,7 @@ mod tests { use super::*; use crate::bridge::dispatch::CoreChannelDataSide; - use crate::bridge::envelope::Status; + use crate::bridge::envelope::{StageVote, Status}; use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ CapturingProposer, elect_data_group_leader, }; @@ -208,7 +208,10 @@ mod tests { peer.pending .insert(txn_id, staged_pending(make_sequenced_txn(30, 0), txn_id)); - peer.resolve_staged_commit(txn_id, &staged_response(Status::Ok, Some(true))); + peer.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); assert_eq!( peer.pending.get(&txn_id).map(|p| p.commit_state), Some(CommitState::AwaitingVerdict) diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs index 9be76f2ec..5b28e8cae 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs @@ -65,6 +65,8 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) enum HaltReason /// A committed txn's write-version record cannot be built. Read-set /// validation reads these versions, so a lost record reports false-valid reads. WriteVersionRecordFailed, + /// A stage response carries no vote the scheduler can read. + StageVoteInvalid, } impl HaltReason { @@ -81,6 +83,7 @@ impl HaltReason { Self::WalAppendFailed => apply_halt_reason::WAL_APPEND_FAILED, Self::MetadataGroupGone => apply_halt_reason::METADATA_GROUP_GONE, Self::WriteVersionRecordFailed => apply_halt_reason::WRITE_VERSION_RECORD_FAILED, + Self::StageVoteInvalid => apply_halt_reason::STAGE_VOTE_INVALID, } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/intake.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/intake.rs index 06cbf77b9..35c30176e 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/intake.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/intake.rs @@ -147,7 +147,7 @@ impl Scheduler { #[cfg(test)] mod tests { use super::*; - use std::collections::BTreeSet; + use std::collections::BTreeMap; use std::sync::Arc; use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; @@ -189,7 +189,7 @@ mod tests { fn blocked_fixture(epoch: u64) -> BlockedTxn { BlockedTxn { txn: make_sequenced_txn(epoch, 0), - keys: BTreeSet::new(), + keys: BTreeMap::new(), // no-determinism: test-only blocked_at timestamp for a fabricated BlockedTxn fixture. blocked_at: Instant::now(), } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/retry.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/retry.rs index a2df9532d..e3e803523 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/retry.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/retry.rs @@ -152,7 +152,7 @@ mod tests { use nodedb_cluster::calvin::{ParticipantVote, SequencerEntry}; use super::*; - use crate::bridge::envelope::Status; + use crate::bridge::envelope::{StageVote, Status}; use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ CapturingProposer, elect_data_group_leader, }; @@ -183,7 +183,10 @@ mod tests { .pending .insert(txn_id, staged_pending(make_sequenced_txn(20, 1), txn_id)); - scheduler.resolve_staged_commit(txn_id, &staged_response(Status::Ok, Some(true))); + scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); assert_eq!(proposer.attempt_count(), 1, "the first proposal is refused"); scheduler.retry_owed_sequencer_entries(); @@ -351,7 +354,10 @@ mod tests { .pending .insert(txn_id, staged_pending(make_sequenced_txn(26, 0), txn_id)); - scheduler.resolve_staged_commit(txn_id, &staged_response(Status::Ok, Some(true))); + scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); scheduler.retry_owed_sequencer_entries(); assert_eq!(proposer.attempt_count(), 0, "a follower casts no vote"); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs index f83ff80e5..54e31ae6a 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs @@ -413,7 +413,6 @@ impl Scheduler { #[cfg(test)] mod tests { use super::*; - use std::collections::BTreeSet; use std::sync::atomic::Ordering; use nodedb_cluster::calvin::CalvinCompletionRegistry; @@ -540,7 +539,7 @@ mod tests { txn_id, crate::control::cluster::calvin::scheduler::driver::types::BlockedTxn { txn: txn.clone(), - keys: BTreeSet::new(), + keys: BTreeMap::new(), // no-determinism: test-only blocked_at timestamp for a fabricated BlockedTxn fixture. blocked_at: Instant::now(), }, diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/staged_vote.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/staged_vote.rs index b0c7f92db..3edf06b3d 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/staged_vote.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/staged_vote.rs @@ -1,11 +1,11 @@ // SPDX-License-Identifier: BUSL-1.1 -//! This participant's local commit vote on a staged static Calvin transaction, -//! derived from the staged executor response. +//! This participant's local commit vote on a staged Calvin transaction, +//! read from the stage vote its executor response carries. use nodedb_cluster::calvin::AbortReason; -use crate::bridge::envelope::{ErrorCode, Response, Status}; +use crate::bridge::envelope::{Response, StageVote, Status}; /// A participant's local verdict on its staged slice. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -19,11 +19,22 @@ pub(super) enum StagedVote { /// A collection the transaction names no longer holds its planned /// incarnation in this replica's catalog. CollectionSuperseded, - /// The stage refused with `OllpRetryRequired`: the state the coordinator - /// predicted moved. The coordinator reads again and retries. + /// The stage refused because the state the coordinator predicted moved. + /// The coordinator reads again and retries. PredictionDrift, } +impl From for StagedVote { + fn from(vote: StageVote) -> Self { + match vote { + StageVote::Commit => Self::Commit, + StageVote::SerializationConflict => Self::SerializationConflict, + StageVote::PredictionDrift => Self::PredictionDrift, + StageVote::ParticipantError => Self::ParticipantError, + } + } +} + impl StagedVote { /// `None` for a commit, the abort cause otherwise. pub(super) fn abort_reason(self) -> Option { @@ -37,30 +48,52 @@ impl StagedVote { } } -/// Derive a local staged vote without ever treating an executor error as a -/// commit, and keep the abort causes apart. A `None` read-set stays -/// affirmative only for a successful dependent-read or active staged response, -/// which has no versioned read-set. -pub(super) fn staged_commit_vote(response: &Response) -> StagedVote { - if response.status != Status::Ok { - if response.error_code.as_deref() == Some(&ErrorCode::OllpRetryRequired) { - return StagedVote::PredictionDrift; +/// A stage response the scheduler cannot read a vote from. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub(super) enum StageVoteError { + /// A stage response that did not fail carries no stage vote. + #[error("stage returned {status:?} without a stage vote")] + Missing { status: Status }, + /// A stage response that did not succeed carries a commit vote. + #[error("stage returned {status:?} with a commit stage vote")] + CommitWithoutSuccess { status: Status }, +} + +/// Read the local vote of a staged slice from its stage response. +/// +/// The response's stage vote maps 1:1. An error response with no vote never +/// reached a stage path: the request expired, or its core is gone or +/// drained. That slice never staged, so it votes `ParticipantError`. +/// +/// A response that did not fail and carries no vote is refused, and so is a +/// commit vote on a response that did not succeed. Neither counts as a commit. +pub(super) fn staged_commit_vote(response: &Response) -> Result { + let status = response.status; + match (response.stage_vote, status) { + (Some(StageVote::Commit), Status::Ok) => Ok(StagedVote::Commit), + (Some(StageVote::Commit), Status::Partial | Status::Error) => { + Err(StageVoteError::CommitWithoutSuccess { status }) } - return StagedVote::ParticipantError; + ( + Some( + vote @ (StageVote::SerializationConflict + | StageVote::PredictionDrift + | StageVote::ParticipantError), + ), + _, + ) => Ok(vote.into()), + (None, Status::Error) => Ok(StagedVote::ParticipantError), + (None, Status::Ok | Status::Partial) => Err(StageVoteError::Missing { status }), } - if response.read_set_valid == Some(false) { - return StagedVote::SerializationConflict; - } - StagedVote::Commit } #[cfg(test)] mod tests { use super::*; - use crate::bridge::envelope::Payload; + use crate::bridge::envelope::{ErrorCode, Payload}; use crate::types::{Lsn, RequestId}; - fn staged_response(status: Status, read_set_valid: Option) -> Response { + fn staged_response(status: Status, stage_vote: Option) -> Response { Response { request_id: RequestId::new(1), status, @@ -69,58 +102,85 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid, + stage_vote, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), } } #[test] - fn executor_error_votes_participant_error_whatever_the_read_set_field_says() { - // The participant never staged, so no read-set was ever validated — - // reporting a serialization conflict here will be a lie. - assert_eq!( - staged_commit_vote(&staged_response(Status::Error, None)), - StagedVote::ParticipantError - ); - assert_eq!( - staged_commit_vote(&staged_response(Status::Error, Some(true))), - StagedVote::ParticipantError - ); - assert_eq!( - staged_commit_vote(&staged_response(Status::Error, Some(false))), - StagedVote::ParticipantError - ); + fn every_stage_vote_maps_one_to_one() { + let cases = [ + (Status::Ok, StageVote::Commit, StagedVote::Commit), + ( + Status::Ok, + StageVote::SerializationConflict, + StagedVote::SerializationConflict, + ), + ( + Status::Error, + StageVote::PredictionDrift, + StagedVote::PredictionDrift, + ), + ( + Status::Error, + StageVote::ParticipantError, + StagedVote::ParticipantError, + ), + ]; + for (status, vote, expected) in cases { + assert_eq!( + staged_commit_vote(&staged_response(status, Some(vote))), + Ok(expected), + "{vote:?} on {status:?}" + ); + } } #[test] - fn an_ollp_retry_stage_error_votes_prediction_drift() { - let mut response = staged_response(Status::Error, None); + fn a_prediction_drift_vote_aborts_with_prediction_drift() { + let mut response = staged_response(Status::Error, Some(StageVote::PredictionDrift)); response.error_code = Some(Box::new(ErrorCode::OllpRetryRequired)); - assert_eq!(staged_commit_vote(&response), StagedVote::PredictionDrift); + let vote = staged_commit_vote(&response); + assert_eq!(vote, Ok(StagedVote::PredictionDrift)); assert_eq!( - StagedVote::PredictionDrift.abort_reason(), - Some(AbortReason::PredictionDrift) + vote.map(StagedVote::abort_reason), + Ok(Some(AbortReason::PredictionDrift)) ); } #[test] - fn stale_read_set_on_a_successful_response_votes_serialization_conflict() { + fn a_successful_stage_response_without_a_vote_is_refused() { assert_eq!( - staged_commit_vote(&staged_response(Status::Ok, Some(false))), - StagedVote::SerializationConflict + staged_commit_vote(&staged_response(Status::Ok, None)), + Err(StageVoteError::Missing { status: Status::Ok }) + ); + assert_eq!( + staged_commit_vote(&staged_response(Status::Partial, None)), + Err(StageVoteError::Missing { + status: Status::Partial + }) ); } #[test] - fn successful_response_with_a_current_or_absent_read_set_votes_commit() { + fn a_commit_vote_on_an_unsuccessful_response_is_refused() { assert_eq!( - staged_commit_vote(&staged_response(Status::Ok, Some(true))), - StagedVote::Commit + staged_commit_vote(&staged_response(Status::Error, Some(StageVote::Commit))), + Err(StageVoteError::CommitWithoutSuccess { + status: Status::Error + }) ); + } + + #[test] + fn an_error_that_never_reached_a_stage_path_votes_participant_error() { + // An expired request answers before any stage handler runs. + let mut response = staged_response(Status::Error, None); + response.error_code = Some(Box::new(ErrorCode::ExpiredBeforeExecution)); assert_eq!( - staged_commit_vote(&staged_response(Status::Ok, None)), - StagedVote::Commit + staged_commit_vote(&response), + Ok(StagedVote::ParticipantError) ); } @@ -139,5 +199,9 @@ mod tests { StagedVote::CollectionSuperseded.abort_reason(), Some(AbortReason::CollectionSuperseded) ); + assert_eq!( + StagedVote::PredictionDrift.abort_reason(), + Some(AbortReason::PredictionDrift) + ); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs index d2a888822..8f6c6673a 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs @@ -20,7 +20,7 @@ use tokio::sync::mpsc; use crate::bridge::dispatch::{BridgeResponse, CoreChannelDataSide, Dispatcher}; use crate::bridge::envelope::{ - Admission, ErrorCode, ExemptReason, Payload, Priority, Request, Response, Status, + Admission, ErrorCode, ExemptReason, Payload, Priority, Request, Response, StageVote, Status, }; use crate::control::cluster::calvin::scheduler::driver::barrier::ReadResultEvent; use crate::control::cluster::calvin::scheduler::driver::core::scheduler::{ @@ -451,8 +451,8 @@ pub(super) fn staged_pending(txn: SequencedTxn, txn_id: TxnId) -> PendingTxn { } } -/// A staged executor `Response` carrying the given status and read-set vote. -pub(super) fn staged_response(status: Status, read_set_valid: Option) -> Response { +/// An executor `Response` carrying the given status and stage vote. +pub(super) fn staged_response(status: Status, stage_vote: Option) -> Response { Response { request_id: RequestId::new(1), status, @@ -461,7 +461,7 @@ pub(super) fn staged_response(status: Status, read_set_valid: Option) -> R payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid, + stage_vote, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs index fb5816af2..4e0d1e198 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs @@ -2,12 +2,12 @@ //! Expansion and decoding helpers for the Calvin scheduler driver. -use std::collections::BTreeSet; +use std::collections::BTreeMap; use std::sync::Arc; use nodedb_cluster::calvin::types::{EngineKeySet, LockKeyWire, SequencedTxn, TxnIdWire}; -use crate::control::cluster::calvin::scheduler::lock_manager::{LockKey, TxnId}; +use crate::control::cluster::calvin::scheduler::lock_manager::{LockKey, LockMode, TxnId}; use nodedb_physical::physical_plan::PhysicalPlan; use nodedb_physical::physical_plan::wire as plan_wire; @@ -42,75 +42,109 @@ pub(super) fn decode_lock_key(wire: &LockKeyWire) -> LockKey { src: *src, dst: *dst, }, + LockKeyWire::Collection { collection } => LockKey::Collection { + collection: Arc::from(collection.as_str()), + }, + LockKeyWire::Unique { + collection, + index, + value, + } => LockKey::Unique { + collection: Arc::from(collection.as_str()), + index: Arc::from(index.as_str()), + value: Arc::from(value.as_slice()), + }, } } -/// Expand the read_set ∪ write_set of a sequenced transaction into a -/// `BTreeSet`. +/// Expand the read set and write set of a sequenced transaction into the +/// moded lock request the scheduler acquires. /// -/// All keys in both the read set and write set are acquired as exclusive -/// (write) locks. -pub(crate) fn expand_rw_set(txn: &SequencedTxn) -> BTreeSet { - let mut keys = BTreeSet::new(); - let add_key_set = |keys: &mut BTreeSet, ks: &EngineKeySet| match ks { - EngineKeySet::Document { - collection, - surrogates, - } - | EngineKeySet::Vector { - collection, - surrogates, - } => { - let coll: Arc = Arc::from(collection.as_str()); +/// - A write row key (surrogate, KV key, edge pair, UNIQUE value) is +/// `Exclusive`, and its collection key is `Intent`. +/// - A write [`EngineKeySet::Collection`] and a write array are `Exclusive` +/// on the collection key. +/// - A read row key is `Shared`, with no collection key. +/// - A read [`EngineKeySet::Collection`] is `Shared` on the collection key. +/// +/// A key named twice takes the merged mode (see [`LockMode::merge`]). +pub(crate) fn expand_rw_set(txn: &SequencedTxn) -> BTreeMap { + let mut keys = BTreeMap::new(); + for ks in &txn.tx_class.read_set.0 { + add_key_set(&mut keys, ks, LockMode::Shared); + } + for ks in &txn.tx_class.write_set.0 { + add_key_set(&mut keys, ks, LockMode::Exclusive); + } + keys +} + +/// Add the keys of `ks` to `keys`. `row_mode` is `Shared` for a read set and +/// `Exclusive` for a write set. +fn add_key_set(keys: &mut BTreeMap, ks: &EngineKeySet, row_mode: LockMode) { + let coll: Arc = Arc::from(ks.collection()); + let whole = LockKey::Collection { + collection: Arc::clone(&coll), + }; + match ks { + EngineKeySet::Document { surrogates, .. } | EngineKeySet::Vector { surrogates, .. } => { for &surrogate in surrogates.iter() { - keys.insert(LockKey::Surrogate { + let key = LockKey::Surrogate { collection: Arc::clone(&coll), surrogate, - }); + }; + request(keys, key, row_mode); } } - EngineKeySet::Kv { - collection, - keys: kv_keys, - } => { - let coll: Arc = Arc::from(collection.as_str()); + EngineKeySet::Kv { keys: kv_keys, .. } => { for k in kv_keys.iter() { - keys.insert(LockKey::Kv { + let key = LockKey::Kv { collection: Arc::clone(&coll), key: Arc::from(k.as_slice()), - }); + }; + request(keys, key, row_mode); } } - EngineKeySet::Edge { - collection, edges, .. - } => { - let coll: Arc = Arc::from(collection.as_str()); + EngineKeySet::Edge { edges, .. } => { for &(src, dst) in edges.iter() { - keys.insert(LockKey::Edge { + let key = LockKey::Edge { collection: Arc::clone(&coll), src, dst, - }); + }; + request(keys, key, row_mode); + } + } + EngineKeySet::Unique { index, values, .. } => { + let index: Arc = Arc::from(index.as_str()); + for value in values.iter() { + let key = LockKey::Unique { + collection: Arc::clone(&coll), + index: Arc::clone(&index), + value: Arc::from(value.as_slice()), + }; + request(keys, key, row_mode); } } // Each participating vShard's lock table holds the whole array on - // that vShard: the collection key every writer of the array takes. - EngineKeySet::Array { collection, .. } => { - keys.insert(LockKey::Surrogate { - collection: Arc::from(collection.as_str()), - surrogate: crate::control::planner::calvin::tx_class::write_keys::COLLECTION_KEY, - }); + // that vShard. A whole-collection key set covers the collection. + EngineKeySet::Array { .. } | EngineKeySet::Collection { .. } => { + request(keys, whole, row_mode); + return; } - }; - - for ks in &txn.tx_class.read_set.0 { - add_key_set(&mut keys, ks); } - for ks in &txn.tx_class.write_set.0 { - add_key_set(&mut keys, ks); + // A row writer announces itself on its collection, so a collection-wide + // writer or a predicate reader of that collection orders against it. + if row_mode == LockMode::Exclusive { + request(keys, whole, LockMode::Intent); } +} - keys +/// Request `key` in `mode`, merged with any mode already requested for it. +fn request(keys: &mut BTreeMap, key: LockKey, mode: LockMode) { + keys.entry(key) + .and_modify(|held| *held = held.merge(mode)) + .or_insert(mode); } /// Decode a `Vec` from the opaque plan bytes stored in a diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs index 7e08b9fe1..e3f237f36 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs @@ -2,12 +2,12 @@ //! In-flight transaction types for the Calvin scheduler driver. -use std::collections::BTreeSet; +use std::collections::BTreeMap; use std::time::Instant; use nodedb_cluster::calvin::types::SequencedTxn; -use super::super::lock_manager::{LockKey, TxnId}; +use super::super::lock_manager::{LockKey, LockMode, TxnId}; /// An in-flight transaction that has been dispatched and is awaiting a /// Data Plane response. @@ -170,7 +170,7 @@ fn reads_whole_collection_at_resolve(plan: &nodedb_physical::physical_plan::Phys /// Commit-resolution state of a staged static Calvin transaction. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(in crate::control::cluster::calvin::scheduler::driver) enum CommitState { - /// Awaiting the validate-and-stage response, whose `read_set_valid` carries + /// Awaiting the validate-and-stage response, whose `stage_vote` carries /// the local commit vote that drives the flush-or-drop decision. Staged, /// The staged txn has cast its local vote and PARKED, awaiting the durable @@ -221,7 +221,8 @@ pub(in crate::control::cluster::calvin::scheduler::driver) enum CommitState { /// A transaction that is blocked on lock acquisition. pub(super) struct BlockedTxn { pub txn: SequencedTxn, - pub keys: BTreeSet, + /// The moded lock request the txn waits on. + pub keys: BTreeMap, /// Wall-clock time at first block (for latency metrics). /// /// `Instant::now()` used for observability only. diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/lock_entry.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/lock_entry.rs index 6b8b53082..314ae76aa 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/lock_entry.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/lock_entry.rs @@ -2,9 +2,8 @@ //! Per-key lock state for the Calvin lock table. //! -//! A [`LockEntry`] holds either a single exclusive holder or a set of shared -//! holders, plus a FIFO waiter queue that carries each waiter's requested -//! [`LockMode`]. +//! A [`LockEntry`] holds one group of compatible holders, plus a FIFO waiter +//! queue that carries each waiter's requested [`LockMode`]. use std::collections::VecDeque; @@ -16,16 +15,53 @@ use super::lock_key::TxnId; /// The mode under which a lock is held or requested. /// -/// `Shared` locks are mutually compatible (many readers); `Exclusive` locks -/// conflict with everything else. +/// Compatibility: +/// +/// | held \ requested | Intent | Shared | Exclusive | +/// |------------------|--------|--------|-----------| +/// | Intent | yes | no | no | +/// | Shared | no | yes | no | +/// | Exclusive | no | no | no | #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum LockMode { + /// A row writer's hold on its collection key. Compatible only with other + /// `Intent` holders, so row writers run together and exclude a predicate + /// reader or a collection-wide writer. + Intent, /// Compatible with other shared holders on the same key. Shared, /// Held by exactly one transaction; excludes all others. Exclusive, } +impl LockMode { + /// Whether a holder in `self` and a holder in `other` can hold one key + /// together. + pub fn compatible(self, other: LockMode) -> bool { + match (self, other) { + (LockMode::Intent, LockMode::Intent) | (LockMode::Shared, LockMode::Shared) => true, + (LockMode::Intent, LockMode::Shared | LockMode::Exclusive) + | (LockMode::Shared, LockMode::Intent | LockMode::Exclusive) + | (LockMode::Exclusive, LockMode::Intent | LockMode::Shared | LockMode::Exclusive) => { + false + } + } + } + + /// The one mode that covers both `self` and `other` for one transaction. + /// + /// Equal modes merge to themselves. Any other pair merges to `Exclusive`: + /// `Shared` plus `Intent` means the transaction both reads the whole + /// collection and writes rows of it. + pub fn merge(self, other: LockMode) -> LockMode { + if self == other { + self + } else { + LockMode::Exclusive + } + } +} + // ── AcquireOutcome ──────────────────────────────────────────────────────────── /// Result of a lock-acquire call. @@ -33,8 +69,9 @@ pub enum LockMode { pub enum AcquireOutcome { /// All requested locks were granted; the transaction is ready to dispatch. Ready, - /// At least one key conflicted with a current holder. The transaction has - /// been enqueued as a waiter on every unavailable key. + /// At least one key conflicted with a current holder or an earlier + /// waiter. The transaction holds every key it could take and waits in + /// FIFO order on every other key. Blocked, } @@ -43,17 +80,54 @@ pub enum AcquireOutcome { /// Per-key lock state. /// /// Invariants maintained by [`LockManager`](super::manager::LockManager): +/// - Every holder holds the key in `mode`. Two modes never mix, because no +/// two distinct modes are compatible. /// - `Exclusive` entries have exactly one holder. -/// - `Shared` entries have one or more holders, all compatible. /// - `waiters` is FIFO; each waiter carries the mode it requested so that /// promotion on release is mode-aware. pub(super) struct LockEntry { /// The mode the current holders hold this lock under. pub(super) mode: LockMode, /// The transactions currently holding this lock. Exclusive: exactly one; - /// Shared: one or more. + /// Shared or Intent: one or more. pub(super) holders: SmallVec<[TxnId; 2]>, /// Transactions waiting for this lock, in FIFO order, each tagged with the /// mode it requested. pub(super) waiters: VecDeque<(TxnId, LockMode)>, } + +#[cfg(test)] +mod tests { + use super::*; + + const ALL: [LockMode; 3] = [LockMode::Intent, LockMode::Shared, LockMode::Exclusive]; + + #[test] + fn compatibility_matrix() { + assert!(LockMode::Intent.compatible(LockMode::Intent)); + assert!(LockMode::Shared.compatible(LockMode::Shared)); + assert!(!LockMode::Intent.compatible(LockMode::Shared)); + assert!(!LockMode::Shared.compatible(LockMode::Intent)); + for mode in ALL { + assert!(!mode.compatible(LockMode::Exclusive)); + assert!(!LockMode::Exclusive.compatible(mode)); + } + } + + #[test] + fn merge_keeps_the_strongest_mode() { + for mode in ALL { + assert_eq!(mode.merge(mode), mode); + assert_eq!(mode.merge(LockMode::Exclusive), LockMode::Exclusive); + assert_eq!(LockMode::Exclusive.merge(mode), LockMode::Exclusive); + } + assert_eq!( + LockMode::Shared.merge(LockMode::Intent), + LockMode::Exclusive + ); + assert_eq!( + LockMode::Intent.merge(LockMode::Shared), + LockMode::Exclusive + ); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/lock_key.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/lock_key.rs index cd1fe5224..c8c5883ab 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/lock_key.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/lock_key.rs @@ -36,6 +36,17 @@ pub enum LockKey { src: u32, dst: u32, }, + /// A whole collection. Writers of single rows hold it `Intent`. Writers + /// of the whole collection hold it `Exclusive`. Predicate readers hold it + /// `Shared`. + Collection { collection: Arc }, + /// One claimed value of one UNIQUE index. Writers that claim the value + /// hold it `Exclusive`, so two claims of one value serialize. + Unique { + collection: Arc, + index: Arc, + value: Arc<[u8]>, + }, } // ── TxnId ───────────────────────────────────────────────────────────────────── diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/acquire.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/acquire.rs index f896c15c9..0d13629bc 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/acquire.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/acquire.rs @@ -1,160 +1,103 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Exclusive lock acquisition and waiter queueing. +//! Moded lock acquisition and FIFO waiter queueing. -use std::collections::{BTreeSet, VecDeque}; +use std::collections::BTreeMap; -use smallvec::smallvec; - -use crate::control::cluster::calvin::scheduler::lock::lock_entry::{ - AcquireOutcome, LockEntry, LockMode, -}; +use crate::control::cluster::calvin::scheduler::lock::lock_entry::{AcquireOutcome, LockMode}; use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId}; -use super::types::{ExclusiveWait, LockManager}; +use super::classify::KeyState; +use super::types::{ConflictResolution, LockManager}; impl LockManager { - /// Attempt to acquire **exclusive** locks on all keys for `txn`. + /// Acquire every key of `keys` for `txn`, each in its own mode. /// - /// If every key is free, already held exclusively by `txn` (promoted from - /// waiter), or held **shared solely by `txn`** (a reservation this txn placed - /// earlier, now upgraded in place to exclusive), records `txn` as sole holder - /// of each key and returns [`AcquireOutcome::Ready`]. + /// The request merges with any request `txn` already waits on. A key the + /// txn names twice takes the merged mode (see [`LockMode::merge`]). /// - /// Otherwise the whole key set is classified under the WOUND-WAIT discipline - /// (see [`Self::wound_or_block`]). `TxnId` order (`(epoch, position)`) is - /// the replicated total order, so "older" means a smaller id and the - /// decision is a pure function of the lock table plus the replicated ids — - /// every replica computes it identically: - /// - Any conflicting holder is **exclusive** → `txn` waits (an exclusive - /// holder is executing/applied work and is never wounded). - /// - All conflicting holders are **shared** reservations and `txn` is older - /// than every one of them → **wound** them all (revoke to plain OCC, no - /// notification) and take every key. Wounding is silent. - /// - Otherwise (`txn` younger than some conflicting shared holder) → wait. + /// When no key conflicts, `txn` takes every key and the call returns + /// [`AcquireOutcome::Ready`]. A key conflicts when an incompatible holder + /// holds it, or when an earlier waiter queues on it. /// - /// On the wait path `txn` is enqueued as an exclusive waiter on every - /// conflicting key and holds none; its key set is stored in `pending_keys` - /// so `release` can promote it atomically when all keys become available. - /// The whole key set is evaluated before any mutation, so acquisition stays - /// all-keys-or-none — the manager never partially wounds and then blocks. - pub fn acquire(&mut self, txn: TxnId, keys: BTreeSet) -> AcquireOutcome { - // First pass: determine whether any key is held by a *different* txn. - // A key already held exclusively by `txn` (promoted via release) or held - // shared solely by `txn` (an earlier reservation) counts as available — - // the former is a no-op re-acquire, the latter a self-upgrade to exclusive. - let all_available = keys.iter().all(|k| { - self.table.get(k).is_none_or(|entry| { - entry.held_exclusively_by(txn) || entry.held_shared_solely_by(txn) - }) - }); - - if all_available { - // Acquire all keys. For keys not yet in the table (free), insert a - // new exclusive entry. For keys already held by this txn (promoted - // waiter), leave the entry unchanged — the waiter queue is intact. - for key in &keys { - match self.table.get_mut(key) { - None => { - self.table.insert( - key.clone(), - LockEntry { - mode: LockMode::Exclusive, - holders: smallvec![txn], - waiters: VecDeque::new(), - }, - ); - } - Some(entry) => { - // A key this txn already holds shared-solely is upgraded - // to exclusive in place (holders is exactly `[txn]`, so no - // holder change and any waiter queue stays intact). A key - // already held exclusively by `txn` is left unchanged. - if entry.held_shared_solely_by(txn) { - entry.mode = LockMode::Exclusive; - } - } + /// A conflicting request is resolved under WOUND-WAIT (see + /// [`Self::resolve_conflict`]). `TxnId` order is the replicated total + /// order, so the decision is identical on every replica: + /// - Every conflicting holder is a younger read reservation in `Shared` + /// mode, and nobody waits on a conflicting key: `txn` revokes those + /// reservations and takes every key. + /// - Otherwise `txn` takes every key it can take now and waits FIFO on the + /// rest. Calls arrive in sequencer order, so every key grants its + /// requesters in sequencer order, whatever order earlier holders + /// release in. + pub fn acquire(&mut self, txn: TxnId, keys: BTreeMap) -> AcquireOutcome { + let mut request = self.pending_keys.remove(&txn).unwrap_or_default(); + for (key, mode) in keys { + request + .entry(key) + .and_modify(|held| *held = held.merge(mode)) + .or_insert(mode); + } + let states: Vec = request + .iter() + .map(|(key, mode)| self.classify(txn, key, *mode)) + .collect(); + + let resolution = if states.contains(&KeyState::Conflict) { + Some(self.resolve_conflict(txn, &request, &states)) + } else { + None + }; + + match resolution { + None => { + for (key, mode) in &request { + self.grant(txn, key, *mode); } + self.finish_ready(txn, request); + AcquireOutcome::Ready } - // Move out of pending (if the txn was previously blocked on this - // same key set) and into held_locks. - self.pending_keys.remove(&txn); - self.held_locks.insert(txn, keys); - return AcquireOutcome::Ready; - } - - // A conflict exists. Classify the WHOLE key set before mutating so the - // wound / block decision is atomic (never partially wound then block). - match self.wound_or_block(txn, &keys) { - ExclusiveWait::Wound => { - // Take every key exclusively. A conflicting shared entry has its - // holders revoked (the wounded readers, all younger than `txn`, - // degrade to plain OCC); `txn` becomes the sole holder while any - // existing waiters remain queued behind it. - for key in &keys { - match self.table.get_mut(key) { - Some(entry) => { - entry.mode = LockMode::Exclusive; - entry.holders.clear(); - entry.holders.push(txn); - } - None => { - self.table.insert( - key.clone(), - LockEntry { - mode: LockMode::Exclusive, - holders: smallvec![txn], - waiters: VecDeque::new(), - }, - ); + Some(ConflictResolution::Wound) => { + for ((key, mode), state) in request.iter().zip(&states) { + match state { + KeyState::Conflict => self.wound(txn, key, *mode), + KeyState::Free | KeyState::Held | KeyState::Join => { + self.grant(txn, key, *mode) } } } - self.pending_keys.remove(&txn); - self.held_locks.insert(txn, keys); + self.finish_ready(txn, request); AcquireOutcome::Ready } - ExclusiveWait::Block => { - // Enqueue as an exclusive waiter on every key held by a - // different txn. - for key in &keys { - if let Some(entry) = self.table.get_mut(key) { - // No conflict on this key means it is held solely by `txn` - // (a shared reservation to be upgraded, or an exclusive - // re-acquire) — leave it untouched; `txn` keeps the key and - // upgrades it once its conflicting keys are free. Same - // predicate as the `all_available` check above. - if entry.held_exclusively_by(txn) || entry.held_shared_solely_by(txn) { - continue; - } - // Real conflict on this key. If `txn` also holds it shared - // (an upgrade that must wait behind an OLDER shared holder), - // drop its own shared hold — degrading that read to plain - // OCC, never worse than today — so the key can drain to - // empty and normal promotion can grant `txn` the exclusive - // lock later. Without this, `txn` would occupy the key - // forever and its own exclusive request could never fire. - entry.holders.retain(|h| *h != txn); - if !entry.has_waiter(txn) { - entry.waiters.push_back((txn, LockMode::Exclusive)); + Some(ConflictResolution::Block) => { + for ((key, mode), state) in request.iter().zip(&states) { + match state { + KeyState::Conflict => self.enqueue(txn, key, *mode), + KeyState::Free | KeyState::Held | KeyState::Join => { + self.grant(txn, key, *mode) } } - // Free keys: no entry exists; the txn acquires them on the - // re-acquire path after all conflicting keys are released. } - // Store the full key set so that release can promote this txn - // atomically once all its keys become available. - self.pending_keys.insert(txn, keys); + self.pending_keys.insert(txn, request); AcquireOutcome::Blocked } } } + + /// Record `txn` as holder of every key of `request`. + pub(super) fn finish_ready(&mut self, txn: TxnId, request: BTreeMap) { + self.held_locks + .entry(txn) + .or_default() + .extend(request.into_keys()); + } } // ── Tests ───────────────────────────────────────────────────────────────────── #[cfg(test)] mod tests { + use std::collections::BTreeSet; use std::sync::Arc; use super::*; @@ -166,8 +109,21 @@ mod tests { } } - fn keyset(names: &[&str]) -> BTreeSet { - names.iter().map(|n| key(n)).collect() + fn coll(name: &str) -> LockKey { + LockKey::Collection { + collection: Arc::from(name), + } + } + + fn exclusive(names: &[&str]) -> BTreeMap { + names + .iter() + .map(|n| (key(n), LockMode::Exclusive)) + .collect() + } + + fn one(key: LockKey, mode: LockMode) -> BTreeMap { + BTreeMap::from([(key, mode)]) } fn txn(epoch: u64, pos: u32) -> TxnId { @@ -177,8 +133,7 @@ mod tests { #[test] fn acquire_free_keys_returns_ready() { let mut lm = LockManager::new(); - let t = txn(1, 0); - let outcome = lm.acquire(t, keyset(&["a", "b"])); + let outcome = lm.acquire(txn(1, 0), exclusive(&["a", "b"])); assert_eq!(outcome, AcquireOutcome::Ready); assert_eq!(lm.lock_count(), 2); } @@ -186,98 +141,191 @@ mod tests { #[test] fn acquire_held_key_returns_blocked_and_enqueues_waiter() { let mut lm = LockManager::new(); - let t1 = txn(1, 0); - let t2 = txn(1, 1); - lm.acquire(t1, keyset(&["x"])); - - let outcome = lm.acquire(t2, keyset(&["x"])); - assert_eq!(outcome, AcquireOutcome::Blocked); + let (t1, t2) = (txn(1, 0), txn(1, 1)); + lm.acquire(t1, exclusive(&["x"])); - // t2 should be in the waiter queue for "x". + assert_eq!(lm.acquire(t2, exclusive(&["x"])), AcquireOutcome::Blocked); assert!(lm.table.get(&key("x")).unwrap().has_waiter(t2)); } #[test] - fn exclusive_waits_on_exclusive() { + fn intent_intent_compatible() { let mut lm = LockManager::new(); - let t1 = txn(1, 0); - let t2 = txn(1, 1); + let (t1, t2) = (txn(1, 0), txn(1, 1)); + assert_eq!( + lm.acquire(t1, one(coll("c"), LockMode::Intent)), + AcquireOutcome::Ready + ); + assert_eq!( + lm.acquire(t2, one(coll("c"), LockMode::Intent)), + AcquireOutcome::Ready, + "two row writers of one collection hold its key together" + ); + let entry = lm.table.get(&coll("c")).unwrap(); + assert_eq!(entry.mode, LockMode::Intent); + assert_eq!(entry.holders.as_slice(), &[t1, t2]); + } - assert_eq!(lm.acquire(t1, keyset(&["k"])), AcquireOutcome::Ready); - assert_eq!(lm.acquire(t2, keyset(&["k"])), AcquireOutcome::Blocked); - assert!(lm.table.get(&key("k")).unwrap().has_waiter(t2)); + #[test] + fn intent_blocks_against_exclusive_and_shared() { + for other in [LockMode::Exclusive, LockMode::Shared] { + // Intent held, the other mode requested. + let mut lm = LockManager::new(); + let (t1, t2) = (txn(1, 0), txn(1, 1)); + assert_eq!( + lm.acquire(t1, one(coll("c"), LockMode::Intent)), + AcquireOutcome::Ready + ); + assert_eq!( + lm.acquire(t2, one(coll("c"), other)), + AcquireOutcome::Blocked, + "{other:?} waits behind an Intent holder" + ); + assert_eq!(lm.release(t1), vec![t2]); + + // The other mode held, Intent requested. + let mut lm = LockManager::new(); + assert_eq!(lm.acquire(t1, one(coll("c"), other)), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(t2, one(coll("c"), LockMode::Intent)), + AcquireOutcome::Blocked, + "Intent waits behind a {other:?} holder" + ); + assert_eq!(lm.release(t1), vec![t2]); + } } #[test] - fn shared_reservation_self_upgrades_to_exclusive() { + fn a_compatible_request_queues_behind_an_earlier_waiter() { let mut lm = LockManager::new(); - let t = txn(1, 0); - - assert_eq!(lm.acquire_shared(t, key("k")), AcquireOutcome::Ready); - // The txn re-acquires its own shared reservation exclusively — this must - // NOT self-deadlock by blocking on its own held key. + let (reader, truncate, writer) = (txn(1, 0), txn(1, 1), txn(1, 2)); assert_eq!( - lm.acquire(t, keyset(&["k"])), - AcquireOutcome::Ready, - "self-upgrade from shared to exclusive must not block" + lm.acquire(reader, one(coll("c"), LockMode::Shared)), + AcquireOutcome::Ready ); - - let entry = lm.table.get(&key("k")).unwrap(); - assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.len(), 1); - assert_eq!(entry.holders[0], t); + assert_eq!( + lm.acquire(truncate, one(coll("c"), LockMode::Exclusive)), + AcquireOutcome::Blocked + ); + // A later reader is compatible with the holder, yet it must not pass + // the truncate sequenced before it. + let late_reader = txn(1, 3); + assert_eq!( + lm.acquire(late_reader, one(coll("c"), LockMode::Shared)), + AcquireOutcome::Blocked + ); + assert_eq!( + lm.acquire(writer, one(coll("c"), LockMode::Intent)), + AcquireOutcome::Blocked + ); + assert_eq!(lm.release(reader), vec![truncate]); + assert_eq!(lm.release(truncate), vec![late_reader]); + assert_eq!(lm.release(late_reader), vec![writer]); } #[test] - fn self_upgrade_with_other_shared_holder_blocks_or_wounds() { - // T_old is older than T_young: T_old's self-upgrade must wound T_young. + fn a_blocked_txn_takes_its_free_keys_so_a_later_txn_queues_behind_it() { let mut lm = LockManager::new(); - let t_old = txn(1, 0); - let t_young = txn(1, 1); + let (first, second, third) = (txn(1, 0), txn(1, 1), txn(1, 2)); + assert_eq!(lm.acquire(first, exclusive(&["a"])), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(second, exclusive(&["a", "b"])), + AcquireOutcome::Blocked + ); + // `b` is free at `second`'s request, so `second` holds it now. A + // later txn on `b` waits for `second`, whatever order releases run in. + assert_eq!( + lm.acquire(third, exclusive(&["b"])), + AcquireOutcome::Blocked + ); + assert_eq!(lm.release(first), vec![second]); + assert_eq!(lm.release(second), vec![third]); + } - assert_eq!(lm.acquire_shared(t_old, key("k")), AcquireOutcome::Ready); - assert_eq!(lm.acquire_shared(t_young, key("k")), AcquireOutcome::Ready); + #[test] + fn the_strongest_mode_wins_within_one_request() { + let mut lm = LockManager::new(); + let t = txn(1, 0); + assert_eq!( + lm.acquire(t, one(coll("c"), LockMode::Shared)), + AcquireOutcome::Ready + ); + // The same txn now writes a row: Shared plus Intent merges to + // Exclusive, held in place by the sole holder. + assert_eq!( + lm.acquire(t, one(coll("c"), LockMode::Intent)), + AcquireOutcome::Ready + ); + assert_eq!(lm.table.get(&coll("c")).unwrap().mode, LockMode::Exclusive); + let other = txn(1, 1); + assert_eq!( + lm.acquire(other, one(coll("c"), LockMode::Shared)), + AcquireOutcome::Blocked + ); + } + #[test] + fn grant_order_is_identical_for_the_same_input_sequence() { + // Every input of the sequence, applied in order: an acquire or a release. + enum Step { + Acquire(TxnId, BTreeMap), + Release(TxnId), + } + let steps = || { + let row = |n: &str| (key(n), LockMode::Exclusive); + let intent = (coll("c"), LockMode::Intent); + vec![ + Step::Acquire(txn(1, 0), BTreeMap::from([intent.clone(), row("a")])), + Step::Acquire(txn(1, 1), one(coll("c"), LockMode::Exclusive)), + Step::Acquire(txn(1, 2), BTreeMap::from([intent.clone(), row("b")])), + Step::Acquire(txn(1, 3), one(coll("c"), LockMode::Shared)), + Step::Acquire(txn(2, 0), BTreeMap::from([intent, row("a")])), + Step::Release(txn(1, 0)), + Step::Release(txn(1, 1)), + Step::Release(txn(1, 2)), + Step::Release(txn(1, 3)), + Step::Release(txn(2, 0)), + ] + }; + let run = || { + let mut lm = LockManager::new(); + let mut granted = Vec::new(); + for step in steps() { + match step { + Step::Acquire(t, keys) => { + if lm.acquire(t, keys) == AcquireOutcome::Ready { + granted.push(t); + } + } + Step::Release(t) => granted.extend(lm.release(t)), + } + } + granted + }; + let first = run(); + assert_eq!(first, run()); assert_eq!( - lm.acquire(t_old, keyset(&["k"])), - AcquireOutcome::Ready, - "the older self-upgrader wounds the younger shared holder" + first, + vec![txn(1, 0), txn(1, 1), txn(1, 2), txn(1, 3), txn(2, 0),], + "every txn is granted in sequencer order" ); - let entry = lm.table.get(&key("k")).unwrap(); - assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.len(), 1); - assert_eq!(entry.holders[0], t_old); + } - // Symmetric case: the YOUNGER of the two self-upgrades and must block. + #[test] + fn shared_reservation_self_upgrades_to_exclusive() { let mut lm = LockManager::new(); - let t_old = txn(1, 0); - let t_young = txn(1, 1); - - assert_eq!(lm.acquire_shared(t_old, key("k")), AcquireOutcome::Ready); - assert_eq!(lm.acquire_shared(t_young, key("k")), AcquireOutcome::Ready); + let t = txn(1, 0); + assert_eq!(lm.acquire_shared(t, key("k")), AcquireOutcome::Ready); assert_eq!( - lm.acquire(t_young, keyset(&["k"])), - AcquireOutcome::Blocked, - "the younger self-upgrader must wait behind the older shared holder" + lm.acquire(t, exclusive(&["k"])), + AcquireOutcome::Ready, + "self-upgrade from shared to exclusive must not block" ); - // t_young drops its own shared hold (degrading to plain OCC) so the key - // can drain to empty and its exclusive request can later be promoted; - // t_old remains the sole shared holder, and t_young is enqueued as an - // exclusive waiter rather than left stuck as a non-waiting holder. - let entry = lm.table.get(&key("k")).unwrap(); - assert_eq!(entry.mode, LockMode::Shared); - assert!(entry.holders.contains(&t_old)); - assert!(!entry.holders.contains(&t_young)); - assert!(entry.has_waiter(t_young)); - - // Once t_old releases, t_young is promoted to sole exclusive holder. - let unblocked = lm.release(t_old); - assert!(unblocked.contains(&t_young)); + let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert_eq!(entry.holders.len(), 1); - assert_eq!(entry.holders[0], t_young); + assert_eq!(entry.holders.as_slice(), &[t]); } #[test] @@ -286,62 +334,26 @@ mod tests { let t = txn(1, 0); let u = txn(1, 1); - // T reserves K1 shared; U holds K2 exclusively. assert_eq!(lm.acquire_shared(t, key("k1")), AcquireOutcome::Ready); - assert_eq!(lm.acquire(u, keyset(&["k2"])), AcquireOutcome::Ready); + assert_eq!(lm.acquire(u, exclusive(&["k2"])), AcquireOutcome::Ready); - // T tries to take both keys exclusively: K2 conflicts with U, so T must - // block on the whole set — and critically must NOT self-deadlock on K1. assert_eq!( - lm.acquire(t, keyset(&["k1", "k2"])), + lm.acquire(t, exclusive(&["k1", "k2"])), AcquireOutcome::Blocked, "conflict on k2 blocks the whole set" ); - // After U releases K2, T's re-acquire succeeds and upgrades K1 in place. - lm.release(u); + assert_eq!(lm.release(u), vec![t]); assert_eq!( - lm.acquire(t, keyset(&["k1", "k2"])), + lm.acquire(t, exclusive(&["k1", "k2"])), AcquireOutcome::Ready, - "once k2 frees up, t acquires both keys exclusively" - ); - let k1 = lm.table.get(&key("k1")).unwrap(); - assert_eq!(k1.mode, LockMode::Exclusive); - assert_eq!(k1.holders.len(), 1); - assert_eq!(k1.holders[0], t); - let k2 = lm.table.get(&key("k2")).unwrap(); - assert_eq!(k2.mode, LockMode::Exclusive); - assert_eq!(k2.holders.len(), 1); - assert_eq!(k2.holders[0], t); - } - - #[test] - fn two_non_conflicting_both_dispatch_immediately() { - let mut lm = LockManager::new(); - - let txn1 = TxnId::new(1, 0); - let txn2 = TxnId::new(1, 1); - - let keys1: BTreeSet = [LockKey::Surrogate { - collection: Arc::from("coll"), - surrogate: 1, - }] - .into(); - let keys2: BTreeSet = [LockKey::Surrogate { - collection: Arc::from("coll"), - surrogate: 2, - }] - .into(); - - let o1 = lm.acquire(txn1, keys1); - let o2 = lm.acquire(txn2, keys2); - - assert_eq!(o1, AcquireOutcome::Ready, "txn1 should be ready"); - assert_eq!( - o2, - AcquireOutcome::Ready, - "txn2 should be ready (disjoint keys)" + "the promoted txn re-acquires its keys as a no-op" ); + for k in ["k1", "k2"] { + let entry = lm.table.get(&key(k)).unwrap(); + assert_eq!(entry.mode, LockMode::Exclusive); + assert_eq!(entry.holders.as_slice(), &[t]); + } } #[test] @@ -352,27 +364,20 @@ mod tests { let pairs = [(2, 0), (1, 1), (3, 0), (1, 0), (2, 1)]; for (epoch, pos) in pairs { let tid = TxnId::new(epoch, pos); - let keys: BTreeSet = [LockKey::Surrogate { - collection: Arc::from(format!("c_{epoch}_{pos}")), - surrogate: epoch as u32 * 10 + pos, - }] - .into(); - let outcome = lm.acquire(tid, keys); - if outcome == AcquireOutcome::Ready { + let keys = one( + LockKey::Surrogate { + collection: Arc::from(format!("c_{epoch}_{pos}")), + surrogate: epoch as u32 * 10 + pos, + }, + LockMode::Exclusive, + ); + if lm.acquire(tid, keys) == AcquireOutcome::Ready { dispatched.push(tid); } } - assert_eq!( - dispatched.len(), - 5, - "all non-conflicting txns should be ready" - ); - - let mut expected = pairs.map(|(e, p)| TxnId::new(e, p)).to_vec(); - expected.sort(); - let mut sorted_dispatched = dispatched.clone(); - sorted_dispatched.sort(); - assert_eq!(sorted_dispatched, expected); + let expected: BTreeSet = pairs.map(|(e, p)| TxnId::new(e, p)).into(); + let got: BTreeSet = dispatched.into_iter().collect(); + assert_eq!(got, expected, "all non-conflicting txns are ready"); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/classify.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/classify.rs new file mode 100644 index 000000000..42492d644 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/classify.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Per-key classification of a lock request, and the grant that installs a +//! requester on a key it can take now. + +use std::collections::VecDeque; + +use smallvec::smallvec; + +use crate::control::cluster::calvin::scheduler::lock::lock_entry::{LockEntry, LockMode}; +use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId}; + +use super::types::LockManager; + +/// Where one requested key stands for one requester. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum KeyState { + /// No entry: the requester can take the key. + Free, + /// The requester already holds the key in a mode that covers the request, + /// or holds it alone and can raise its mode in place. + Held, + /// Other holders hold the key in a compatible mode and nobody waits: the + /// requester can join them. + Join, + /// The requester must wait: an incompatible holder, an earlier waiter, or + /// its own earlier wait on this key. + Conflict, +} + +impl LockManager { + /// Classify `key` requested in `mode` by `txn`. + pub(super) fn classify(&self, txn: TxnId, key: &LockKey, mode: LockMode) -> KeyState { + let Some(entry) = self.table.get(key) else { + return KeyState::Free; + }; + if entry.holders.contains(&txn) { + if entry.mode.merge(mode) == entry.mode || entry.held_solely_by(txn) { + return KeyState::Held; + } + return KeyState::Conflict; + } + if entry.has_waiter(txn) { + return KeyState::Conflict; + } + if entry.mode.compatible(mode) && entry.waiters.is_empty() { + return KeyState::Join; + } + KeyState::Conflict + } + + /// Install `txn` on `key` in `mode`. The caller classified the key as + /// [`KeyState::Free`], [`KeyState::Held`], or [`KeyState::Join`]. + pub(super) fn grant(&mut self, txn: TxnId, key: &LockKey, mode: LockMode) { + match self.table.get_mut(key) { + None => { + self.table.insert( + key.clone(), + LockEntry { + mode, + holders: smallvec![txn], + waiters: VecDeque::new(), + }, + ); + } + Some(entry) => { + if entry.holders.contains(&txn) { + // A sole holder raises its mode in place. A co-holder only + // reaches here when its mode already covers the request. + if entry.held_solely_by(txn) { + entry.mode = entry.mode.merge(mode); + } + } else { + entry.holders.push(txn); + } + } + } + } + + /// Enqueue `txn` as a waiter on a [`KeyState::Conflict`] key. + /// + /// A txn that holds the key and needs a stronger mode drops its own hold + /// and waits for the merged mode. Its read degrades to plain OCC until the + /// grant, and the key can drain so the grant can fire. + pub(super) fn enqueue(&mut self, txn: TxnId, key: &LockKey, mode: LockMode) { + let Some(entry) = self.table.get_mut(key) else { + return; + }; + if entry.has_waiter(txn) { + return; + } + let wanted = if entry.holders.contains(&txn) { + entry.holders.retain(|h| *h != txn); + entry.mode.merge(mode) + } else { + mode + }; + entry.waiters.push_back((txn, wanted)); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs index ef9f4e13d..dc442031d 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs @@ -3,8 +3,9 @@ //! Read-only inspection of lock manager state — readiness checks and the //! test-only counters used to assert on table/holder sizes. -use std::collections::BTreeSet; +use std::collections::BTreeMap; +use crate::control::cluster::calvin::scheduler::lock::lock_entry::LockMode; use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId}; use super::types::LockManager; @@ -15,14 +16,14 @@ impl LockManager { /// A transaction is ready when for every key in its key set, the key is /// either: /// - Not present in the lock table (free), or - /// - Present in the lock table with `txn` among the current holders - /// (shared or exclusive). + /// - Present in the lock table with `txn` among the current holders, in + /// any mode. /// /// This is called after `release` returns `txn_id` in the unblocked set. - /// If `is_ready` returns `true`, the caller calls `acquire` again which - /// will succeed on the all-available path (because the waiter was promoted). - pub fn is_ready(&self, txn: TxnId, keys: &BTreeSet) -> bool { - keys.iter().all(|key| { + /// If `is_ready` returns `true`, the caller calls `acquire` again, which + /// grants nothing new because the waiter was promoted. + pub fn is_ready(&self, txn: TxnId, keys: &BTreeMap) -> bool { + keys.keys().all(|key| { match self.table.get(key) { None => true, // key is free Some(entry) => entry.holders.contains(&txn), // txn is a current holder @@ -69,8 +70,11 @@ mod tests { } } - fn keyset(names: &[&str]) -> BTreeSet { - names.iter().map(|n| key(n)).collect() + fn keyset(names: &[&str]) -> BTreeMap { + names + .iter() + .map(|n| (key(n), LockMode::Exclusive)) + .collect() } fn txn(epoch: u64, pos: u32) -> TxnId { diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs index e59e665b9..79746d7be 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs @@ -7,11 +7,11 @@ //! The lock manager provides a deterministic, totally-ordered lock table over //! per-key entries keyed by //! [`LockKey`](crate::control::cluster::calvin::scheduler::lock::LockKey). -//! Locks come in two modes: `Exclusive` (one holder, excludes all others) and -//! `Shared` (many compatible holders). The Calvin batch acquire path takes -//! every key in a transaction's `read_set ∪ write_set` as an `Exclusive` -//! lock; single-key `Shared` locks are available via -//! [`LockManager::acquire_shared`]. +//! Locks come in three modes (see +//! [`LockMode`](crate::control::cluster::calvin::scheduler::lock::LockMode)): +//! `Exclusive` (one holder), `Shared` (many readers), and `Intent` (many row +//! writers of one collection). A transaction acquires one map of key to mode. +//! Read reservations take one `Shared` key via [`LockManager::acquire_shared`]. //! //! # Determinism //! @@ -20,15 +20,17 @@ //! requirement, not a style preference. //! //! Split by concern: -//! - [`types`]: the lock table struct and its internal decision enums. -//! - [`acquire`]: exclusive lock acquisition and waiter queueing. +//! - [`types`]: the lock table struct and its internal decision enum. +//! - [`classify`]: per-key request classification and grants. +//! - [`acquire`]: moded lock acquisition and FIFO waiter queueing. //! - [`wound_wait`]: shared-lock reservations and wound-wait conflict //! resolution. -//! - [`release`]: lock release and FIFO/shared waiter promotion. -//! - [`try_acquire`]: the non-blocking exclusive fast path. +//! - [`release`]: lock release and mode-aware FIFO waiter promotion. +//! - [`try_acquire`]: the non-blocking fast path. //! - [`introspection`]: readiness checks and test counters. mod acquire; +mod classify; mod introspection; mod release; mod try_acquire; diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs index 1ba0550d5..302d9129a 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs @@ -1,113 +1,82 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Lock release and FIFO/shared waiter promotion. +//! Lock release and FIFO, mode-aware waiter promotion. use std::collections::BTreeSet; -use crate::control::cluster::calvin::scheduler::lock::lock_entry::LockMode; use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId}; -use super::types::{LockManager, Promotion}; +use super::types::LockManager; impl LockManager { - /// Release all locks held by `txn`. + /// Release every lock `txn` holds and every wait it queues. /// - /// `txn` is removed from every entry's holder set. When an entry's holders - /// drain to empty, its FIFO waiters are promoted mode-aware: a leading run - /// of shared waiters is promoted together, or a single leading exclusive - /// waiter is promoted alone. A waiter that becomes holder on ALL its - /// pending keys is moved from `pending_keys` to `held_locks` immediately. + /// `txn` leaves the holders and the waiters of every key it held or + /// requested. Each such key then promotes its waiters (see + /// [`Self::promote_waiters`]). A waiter that now holds ALL its requested + /// keys moves from `pending_keys` to `held_locks`. /// - /// Returns the set of `TxnId`s that have been fully promoted (i.e. moved - /// into `held_locks`). The caller may use this list to dispatch those - /// transactions. + /// Returns the fully-promoted `TxnId`s in ascending order. The caller + /// dispatches them. pub fn release(&mut self, txn: TxnId) -> Vec { - let held = match self.held_locks.remove(&txn) { - Some(h) => h, - None => return Vec::new(), - }; + let mut keys: BTreeSet = self.held_locks.remove(&txn).unwrap_or_default(); + if let Some(pending) = self.pending_keys.remove(&txn) { + keys.extend(pending.into_keys()); + } let mut newly_promoted: BTreeSet = BTreeSet::new(); - - for key in &held { - // Drop `txn` from this key's holders. If other (shared) holders - // remain, the key stays held and there is nothing to promote. - let now_empty = match self.table.get_mut(key) { - Some(entry) => { - entry.holders.retain(|h| *h != txn); - entry.holders.is_empty() - } - None => continue, + for key in &keys { + let Some(entry) = self.table.get_mut(key) else { + continue; }; - if now_empty { - self.promote_waiters(key, &mut newly_promoted); - } + entry.holders.retain(|h| *h != txn); + entry.waiters.retain(|(w, _)| *w != txn); + self.promote_waiters(key, &mut newly_promoted); } - newly_promoted.into_iter().collect() } - /// Promote the front of `key`'s waiter queue after its holders drained. + /// Grant the front of `key`'s waiter queue while it is compatible with the + /// current holders, in FIFO order. /// - /// A leading run of shared waiters is granted together; a single leading - /// exclusive waiter is granted alone; an empty queue frees the entry. Any - /// promoted txn that is now holder on all of its pending keys is moved into - /// `held_locks` and recorded in `newly_promoted`. + /// A key with no holders grants its first waiter in that waiter's mode, + /// then every following waiter compatible with it. A key with neither + /// holders nor waiters is removed. Any granted txn that now holds all of + /// its requested keys moves into `held_locks` and joins `newly_promoted`. fn promote_waiters(&mut self, key: &LockKey, newly_promoted: &mut BTreeSet) { - // Decide the promotion inside a scoped borrow so the readiness sweep - // below can re-borrow the table. - let decision = match self.table.get_mut(key) { - Some(entry) => match entry.waiters.front().map(|(_, mode)| *mode) { - None => Promotion::Freed, - Some(LockMode::Exclusive) => match entry.waiters.pop_front() { - Some((next, _)) => { - entry.mode = LockMode::Exclusive; - entry.holders.clear(); - entry.holders.push(next); - Promotion::Promoted(vec![next]) - } - None => Promotion::Freed, - }, - Some(LockMode::Shared) => { - entry.mode = LockMode::Shared; - entry.holders.clear(); - let mut promoted = Vec::new(); - while matches!(entry.waiters.front(), Some((_, LockMode::Shared))) { - if let Some((next, _)) = entry.waiters.pop_front() { - entry.holders.push(next); - promoted.push(next); - } - } - Promotion::Promoted(promoted) - } - }, - None => return, + let mut granted: Vec = Vec::new(); + let Some(entry) = self.table.get_mut(key) else { + return; }; - - let promoted = match decision { - Promotion::Freed => { - self.table.remove(key); - return; + while let Some(&(next, mode)) = entry.waiters.front() { + if !entry.holders.is_empty() && !entry.mode.compatible(mode) { + break; } - Promotion::Promoted(promoted) => promoted, - }; + entry.waiters.pop_front(); + if entry.holders.is_empty() { + entry.mode = mode; + } + entry.holders.push(next); + granted.push(next); + } + if entry.holders.is_empty() && entry.waiters.is_empty() { + self.table.remove(key); + } - // For each promoted txn, check whether it is now holder on ALL of its - // pending keys. If so, it is fully ready — move to held_locks. Remove - // first (rather than `get` + a follow-up `remove`) so there is no - // unwrap/expect on a "just confirmed Some" invariant: the owned - // `pending` set is reinserted on the not-yet-ready path. - for next in promoted { - if let Some(pending) = self.pending_keys.remove(&next) { - let all_held = pending - .iter() - .all(|k| self.table.get(k).is_none_or(|e| e.holders.contains(&next))); - if all_held { - self.held_locks.insert(next, pending); - newly_promoted.insert(next); - } else { - self.pending_keys.insert(next, pending); - } + for next in granted { + let Some(pending) = self.pending_keys.remove(&next) else { + continue; + }; + let all_held = pending.keys().all(|k| { + self.table + .get(k) + .is_some_and(|entry| entry.holders.contains(&next)) + }); + if all_held { + self.finish_ready(next, pending); + newly_promoted.insert(next); + } else { + self.pending_keys.insert(next, pending); } } } @@ -119,8 +88,10 @@ impl LockManager { mod tests { use std::sync::Arc; + use std::collections::BTreeMap; + use super::*; - use crate::control::cluster::calvin::scheduler::lock::lock_entry::AcquireOutcome; + use crate::control::cluster::calvin::scheduler::lock::lock_entry::{AcquireOutcome, LockMode}; fn key(name: &str) -> LockKey { LockKey::Surrogate { @@ -129,8 +100,11 @@ mod tests { } } - fn keyset(names: &[&str]) -> BTreeSet { - names.iter().map(|n| key(n)).collect() + fn keyset(names: &[&str]) -> BTreeMap { + names + .iter() + .map(|n| (key(n), LockMode::Exclusive)) + .collect() } fn txn(epoch: u64, pos: u32) -> TxnId { @@ -290,10 +264,13 @@ mod tests { let txn1 = TxnId::new(1, 0); let txn2 = TxnId::new(1, 1); - let shared_key: BTreeSet = [LockKey::Surrogate { - collection: Arc::from("coll"), - surrogate: 42, - }] + let shared_key: BTreeMap = [( + LockKey::Surrogate { + collection: Arc::from("coll"), + surrogate: 42, + }, + LockMode::Exclusive, + )] .into(); let o1 = lm.acquire(txn1, shared_key.clone()); @@ -315,10 +292,13 @@ mod tests { let txn_n = TxnId::new(1, 0); let txn_n1 = TxnId::new(2, 0); - let key_k: BTreeSet = [LockKey::Surrogate { - collection: Arc::from("orders"), - surrogate: 100, - }] + let key_k: BTreeMap = [( + LockKey::Surrogate { + collection: Arc::from("orders"), + surrogate: 100, + }, + LockMode::Exclusive, + )] .into(); let o1 = lm.acquire(txn_n, key_k.clone()); diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/try_acquire.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/try_acquire.rs index 0dee86f99..1455a67a9 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/try_acquire.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/try_acquire.rs @@ -1,18 +1,22 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Non-blocking exclusive acquire fast path. +//! Non-blocking acquire fast path. -use std::collections::BTreeSet; +use std::collections::BTreeMap; -use crate::control::cluster::calvin::scheduler::lock::lock_entry::AcquireOutcome; +use crate::control::cluster::calvin::scheduler::lock::lock_entry::LockMode; use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId}; +use super::classify::KeyState; use super::types::LockManager; impl LockManager { - /// Non-blocking exclusive acquire: take all `keys` for `txn` iff every one is - /// free (or already held by `txn`), returning `true`; otherwise return - /// `false` WITHOUT enqueuing a waiter or recording any pending state. + /// Non-blocking acquire: take all `keys` for `txn` iff none conflicts, + /// returning `true`; otherwise return `false` WITHOUT enqueuing a waiter or + /// recording any pending state. + /// + /// A key is taken when it is free, already held by `txn`, or held in a + /// compatible mode with nobody waiting (see [`LockMode::compatible`]). /// /// This is the fast path's probe. Unlike [`acquire`](Self::acquire), the /// contended (`false`) path touches NOTHING — no holder, no `pending_keys`, @@ -21,20 +25,42 @@ impl LockManager { /// leaves an orphaned waiter that a later `release` would promote to an /// unowned holder. It also never perturbs the FIFO ordering that Calvin /// transactions depend on. - pub fn try_acquire(&mut self, txn: TxnId, keys: BTreeSet) -> bool { - if !self.is_ready(txn, &keys) { - // Contended: leave the table, waiter queues, and pending_keys - // completely untouched. + pub fn try_acquire(&mut self, txn: TxnId, keys: BTreeMap) -> bool { + let contended = keys + .iter() + .any(|(key, mode)| self.classify(txn, key, *mode) == KeyState::Conflict); + if contended { return false; } - // Every key is free or already held by `txn`, so `acquire` takes its - // all-available path — it inserts the holder and never enqueues. - let outcome = self.acquire(txn, keys); - debug_assert_eq!( - outcome, - AcquireOutcome::Ready, - "try_acquire: is_ready was true but acquire returned Blocked" - ); + for (key, mode) in &keys { + self.grant(txn, key, *mode); + } + self.finish_ready(txn, keys); true } } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + + fn coll() -> LockKey { + LockKey::Collection { + collection: Arc::from("c"), + } + } + + #[test] + fn try_acquire_joins_compatible_holders_and_refuses_conflicts() { + let mut lm = LockManager::new(); + let (a, b, c) = (TxnId::new(1, 0), TxnId::new(1, 1), TxnId::new(1, 2)); + assert!(lm.try_acquire(a, BTreeMap::from([(coll(), LockMode::Intent)]))); + assert!(lm.try_acquire(b, BTreeMap::from([(coll(), LockMode::Intent)]))); + assert!(!lm.try_acquire(c, BTreeMap::from([(coll(), LockMode::Exclusive)]))); + let entry = lm.table.get(&coll()).unwrap(); + assert_eq!(entry.holders.as_slice(), &[a, b]); + assert!(entry.waiters.is_empty(), "a refused probe leaves no waiter"); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs index 93e6862bd..2de091c68 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 -//! The lock table struct, its internal decision enums, and the small -//! per-entry predicates the acquire/release paths share. +//! The lock table struct and the small per-entry predicates the +//! acquire/release paths share. use std::collections::{BTreeMap, BTreeSet}; @@ -15,11 +15,12 @@ use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId} /// /// # Key sets tracked per transaction /// -/// - `held_locks`: key sets for transactions that are a current holder on ALL -/// their keys and are actively executing (i.e. dispatched to the Data Plane). -/// - `pending_keys`: key sets for transactions that are blocked waiting for at -/// least one key. When `release` promotes a blocked txn to holder on every -/// one of its keys, the entry moves from `pending_keys` to `held_locks`. +/// - `held_locks`: keys of transactions that hold ALL their requested keys and +/// are actively executing (i.e. dispatched to the Data Plane). +/// - `pending_keys`: the full request of a transaction that waits for at least +/// one key. It already holds every other key of the request. When `release` +/// grants its last waited key, the entry moves from `pending_keys` to +/// `held_locks`. pub struct LockManager { /// Per-key lock entries. Uses `BTreeMap` for deterministic iteration. /// Visible to the whole `lock` module so the sibling `reap` module can @@ -30,39 +31,24 @@ pub struct LockManager { /// Visible to the whole `lock` module — see `table`. pub(in crate::control::cluster::calvin::scheduler::lock) held_locks: BTreeMap>, - /// Key sets for **blocked** (not-yet-dispatched) txns. Populated when - /// `acquire` returns `Blocked`; cleared (moved to `held_locks`) when all - /// keys have been acquired on the promotion path inside `release`. - pub(super) pending_keys: BTreeMap>, + /// Full requests of **blocked** (not-yet-dispatched) txns. Populated when + /// `acquire` returns `Blocked`; moved to `held_locks` when the last waited + /// key is granted on the promotion path inside `release`. + pub(in crate::control::cluster::calvin::scheduler::lock) pending_keys: + BTreeMap>, } -/// Outcome of inspecting a single key during [`LockManager::acquire_shared`]. -pub(super) enum SharedGrant { - /// The shared lock was granted (key was free or already held shared). - Granted, - /// The key is held exclusively by another txn; the request was enqueued. - Blocked, -} - -/// The wound-wait decision for an exclusive requester that meets a conflict. -pub(super) enum ExclusiveWait { - /// Every conflicting holder is a shared reservation and the requester is - /// older than all of them: wound (revoke) those shared holders and proceed. +/// How a requester resolves a request that conflicts on at least one key. +pub(super) enum ConflictResolution { + /// Every conflicting holder is a younger read reservation and no other + /// transaction waits on a conflicting key: revoke those reservations and + /// take every key. Wound, - /// The requester must block: a conflicting holder is exclusive, or the - /// requester is younger than some conflicting shared holder. + /// The requester waits: a conflicting holder is a transaction, an older + /// reservation, or an earlier waiter queues ahead of it. Block, } -/// The waiters promoted off one key when its holders drained, together with the -/// action to take on the now-empty entry. -pub(super) enum Promotion { - /// No waiters remained; the entry should be removed entirely. - Freed, - /// These waiters were installed as the new holders. - Promoted(Vec), -} - impl LockManager { /// Create an empty lock manager. pub fn new() -> Self { @@ -81,17 +67,9 @@ impl Default for LockManager { } impl LockEntry { - /// Whether this entry is held exclusively by exactly `txn` (the self - /// re-acquire case on the exclusive path). - pub(super) fn held_exclusively_by(&self, txn: TxnId) -> bool { - self.mode == LockMode::Exclusive && self.holders.len() == 1 && self.holders[0] == txn - } - - /// Whether this entry is held **shared** by exactly `txn` and no one else — - /// the self-upgrade case: `txn` may take the key exclusively because it is - /// the sole current holder. - pub(super) fn held_shared_solely_by(&self, txn: TxnId) -> bool { - self.mode == LockMode::Shared && self.holders.len() == 1 && self.holders[0] == txn + /// Whether `txn` is the only holder of this entry. + pub(super) fn held_solely_by(&self, txn: TxnId) -> bool { + self.holders.len() == 1 && self.holders[0] == txn } /// Whether `txn` is already enqueued as a waiter on this entry. diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/wound_wait.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/wound_wait.rs index 2a77b1a45..00d2579c3 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/wound_wait.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/wound_wait.rs @@ -1,119 +1,78 @@ // SPDX-License-Identifier: BUSL-1.1 //! Shared-lock reservations and the wound-wait conflict resolution used by -//! exclusive acquisition. +//! acquisition. -use std::collections::btree_map::Entry; -use std::collections::{BTreeSet, VecDeque}; +use std::collections::BTreeMap; -use smallvec::smallvec; - -use crate::control::cluster::calvin::scheduler::lock::lock_entry::{ - AcquireOutcome, LockEntry, LockMode, -}; +use crate::control::cluster::calvin::scheduler::lock::lock_entry::{AcquireOutcome, LockMode}; use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId}; -use super::types::{ExclusiveWait, LockManager, SharedGrant}; +use super::classify::KeyState; +use super::types::{ConflictResolution, LockManager}; impl LockManager { - /// Classify the wound-wait decision for an exclusive requester `txn` over - /// `keys`, given that at least one key already conflicts. - /// - /// Pure read over the lock table: any exclusive conflict forces - /// [`ExclusiveWait::Block`] (an exclusive holder is never wounded, so a mix - /// of exclusive and shared conflicts blocks too). Otherwise all conflicting - /// holders are shared reservations, and `txn` wounds them only when it is - /// older than every one (`txn < h` for each conflicting shared holder `h`); - /// if it is younger than any, it blocks. A key held only by `txn` itself is - /// not a conflict. - pub(super) fn wound_or_block(&self, txn: TxnId, keys: &BTreeSet) -> ExclusiveWait { - let mut shared_conflicts: Vec = Vec::new(); - for key in keys { - if let Some(entry) = self.table.get(key) { - match entry.mode { - LockMode::Exclusive => { - // Exclusive entries have exactly one holder; a holder - // other than `txn` is an exclusive conflict. - if !entry.holders.contains(&txn) { - return ExclusiveWait::Block; - } - } - LockMode::Shared => { - for holder in &entry.holders { - if *holder != txn { - shared_conflicts.push(*holder); - } - } - } - } - } - } - // Wound only when there is a shared conflict AND `txn` is older than - // every conflicting shared holder; otherwise block. `shared_conflicts` - // only ever holds *other* txns' shared holders (a key held shared solely - // by `txn` never reaches here — it takes the self-upgrade path in - // `acquire`), so an empty set here means every conflict was exclusive. - if !shared_conflicts.is_empty() && shared_conflicts.iter().all(|holder| txn < *holder) { - ExclusiveWait::Wound - } else { - ExclusiveWait::Block - } - } - - /// Attempt to acquire a **shared** lock on a single `key` for `txn`. + /// Decide how `txn` resolves a request with at least one conflicting key. /// - /// - Key free → create a shared entry holding `txn`, return - /// [`AcquireOutcome::Ready`]. - /// - Key held shared → add `txn` to the holders, return - /// [`AcquireOutcome::Ready`]. - /// - Key held exclusively by another txn → enqueue `txn` as a shared waiter - /// (FIFO) and return [`AcquireOutcome::Blocked`]. + /// Pure read over the lock table. `txn` wounds only when every conflicting + /// key meets all of these: + /// - nobody waits on the key, so no earlier request is passed; + /// - every other holder holds the key `Shared`; + /// - every other holder is a read reservation (`TxnId::is_reservation`) + /// younger than `txn`. /// - /// A shared request that meets an exclusive holder blocks FIFO for now; - /// wound-wait priority resolution lands in a following change. - pub fn acquire_shared(&mut self, txn: TxnId, key: LockKey) -> AcquireOutcome { - // Inspect / mutate the entry via the `Entry` API (which takes the key by - // value, sidestepping a get-then-insert borrow conflict) inside a scoped - // borrow so the map-level bookkeeping below can re-borrow `self`. - let grant = match self.table.entry(key.clone()) { - Entry::Vacant(slot) => { - slot.insert(LockEntry { - mode: LockMode::Shared, - holders: smallvec![txn], - waiters: VecDeque::new(), - }); - SharedGrant::Granted + /// A transaction holder is executing work and is never wounded, whatever + /// its mode. Otherwise `txn` blocks. + pub(super) fn resolve_conflict( + &self, + txn: TxnId, + request: &BTreeMap, + states: &[KeyState], + ) -> ConflictResolution { + for (key, state) in request.keys().zip(states) { + if *state != KeyState::Conflict { + continue; } - Entry::Occupied(mut slot) => { - let entry = slot.get_mut(); - if entry.mode == LockMode::Shared { - if !entry.holders.contains(&txn) { - entry.holders.push(txn); - } - SharedGrant::Granted - } else { - // Held exclusively by another txn: block FIFO. - if !entry.has_waiter(txn) { - entry.waiters.push_back((txn, LockMode::Shared)); - } - SharedGrant::Blocked - } + let Some(entry) = self.table.get(key) else { + return ConflictResolution::Block; + }; + if !entry.waiters.is_empty() || entry.mode != LockMode::Shared { + return ConflictResolution::Block; } - }; - - match grant { - SharedGrant::Granted => { - self.pending_keys.remove(&txn); - self.held_locks.entry(txn).or_default().insert(key); - AcquireOutcome::Ready + let mut others = entry.holders.iter().filter(|h| **h != txn).peekable(); + if others.peek().is_none() { + return ConflictResolution::Block; } - SharedGrant::Blocked => { - let mut pending = BTreeSet::new(); - pending.insert(key); - self.pending_keys.insert(txn, pending); - AcquireOutcome::Blocked + if others.any(|holder| !holder.is_reservation() || txn >= *holder) { + return ConflictResolution::Block; } } + ConflictResolution::Wound + } + + /// Revoke every other holder of `key` and make `txn` its sole holder in + /// `mode`, merged with any mode `txn` already held it in. The revoked + /// reservations degrade to plain OCC, with no notification. Waiters stay + /// queued. + pub(super) fn wound(&mut self, txn: TxnId, key: &LockKey, mode: LockMode) { + let Some(entry) = self.table.get_mut(key) else { + return; + }; + let wanted = if entry.holders.contains(&txn) { + entry.mode.merge(mode) + } else { + mode + }; + entry.holders.clear(); + entry.holders.push(txn); + entry.mode = wanted; + } + + /// Acquire a **shared** lock on a single `key` for `txn`: a read + /// reservation. Same rules as [`Self::acquire`]. A reservation that + /// meets an incompatible holder or an earlier waiter waits FIFO. + pub fn acquire_shared(&mut self, txn: TxnId, key: LockKey) -> AcquireOutcome { + self.acquire(txn, BTreeMap::from([(key, LockMode::Shared)])) } } @@ -132,182 +91,226 @@ mod tests { } } - fn keyset(names: &[&str]) -> BTreeSet { - names.iter().map(|n| key(n)).collect() + fn exclusive(names: &[&str]) -> BTreeMap { + names + .iter() + .map(|n| (key(n), LockMode::Exclusive)) + .collect() } fn txn(epoch: u64, pos: u32) -> TxnId { TxnId::new(epoch, pos) } + /// A read-reservation owner: position `n` of the reservation band. + fn reservation(epoch: u64, n: u32) -> TxnId { + TxnId::new(epoch, TxnId::RESERVATION_POSITION_BAND + n) + } + #[test] fn shared_shared_compatible() { let mut lm = LockManager::new(); - let t1 = txn(1, 0); - let t2 = txn(1, 1); + let (r1, r2) = (reservation(1, 0), reservation(1, 1)); - assert_eq!(lm.acquire_shared(t1, key("s")), AcquireOutcome::Ready); - assert_eq!(lm.acquire_shared(t2, key("s")), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(r1, key("s")), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(r2, key("s")), AcquireOutcome::Ready); let entry = lm.table.get(&key("s")).unwrap(); assert_eq!(entry.mode, LockMode::Shared); - assert!(entry.holders.contains(&t1)); - assert!(entry.holders.contains(&t2)); + assert!(entry.holders.contains(&r1)); + assert!(entry.holders.contains(&r2)); } #[test] - fn shared_blocks_exclusive() { + fn a_shared_transaction_holder_is_never_wounded() { let mut lm = LockManager::new(); - let t1 = txn(1, 0); - let t2 = txn(1, 1); + let reader = txn(1, 2); + let writer = txn(1, 1); // older than the reader - assert_eq!(lm.acquire_shared(t1, key("k")), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(reader, key("k")), AcquireOutcome::Ready); assert_eq!( - lm.acquire(t2, keyset(&["k"])), + lm.acquire(writer, exclusive(&["k"])), AcquireOutcome::Blocked, - "an exclusive request must block behind a shared holder" + "a transaction's read lock is executing work, not a reservation" ); - assert!(lm.table.get(&key("k")).unwrap().has_waiter(t2)); + assert!(lm.table.get(&key("k")).unwrap().has_waiter(writer)); } #[test] fn exclusive_blocks_shared() { let mut lm = LockManager::new(); let t1 = txn(1, 0); - let t2 = txn(1, 1); + let r = reservation(1, 0); - assert_eq!(lm.acquire(t1, keyset(&["k"])), AcquireOutcome::Ready); + assert_eq!(lm.acquire(t1, exclusive(&["k"])), AcquireOutcome::Ready); assert_eq!( - lm.acquire_shared(t2, key("k")), + lm.acquire_shared(r, key("k")), AcquireOutcome::Blocked, "a shared request must block behind an exclusive holder" ); - assert!(lm.table.get(&key("k")).unwrap().has_waiter(t2)); + assert!(lm.table.get(&key("k")).unwrap().has_waiter(r)); } #[test] fn older_writer_wounds_shared() { let mut lm = LockManager::new(); - let t2 = txn(1, 2); // shared holder - let t1 = txn(1, 1); // exclusive requester, older than t2 + let r = reservation(1, 0); + let writer = txn(1, 1); // older than every reservation of epoch 1 - assert_eq!(lm.acquire_shared(t2, key("k")), AcquireOutcome::Ready); - // The older writer wounds the younger shared holder and proceeds. - assert_eq!(lm.acquire(t1, keyset(&["k"])), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(r, key("k")), AcquireOutcome::Ready); + assert_eq!(lm.acquire(writer, exclusive(&["k"])), AcquireOutcome::Ready); let entry = lm.table.get(&key("k")).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert!(entry.holders.contains(&t1), "R is now the exclusive holder"); - assert!( - !entry.holders.contains(&t2), - "the wounded shared holder is gone" + assert_eq!(entry.holders.as_slice(), &[writer]); + } + + #[test] + fn older_intent_writer_wounds_a_collection_reservation() { + let mut lm = LockManager::new(); + let r = reservation(1, 0); + let writer = txn(1, 1); + let coll = LockKey::Collection { + collection: Arc::from("c"), + }; + + assert_eq!(lm.acquire_shared(r, coll.clone()), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(writer, BTreeMap::from([(coll.clone(), LockMode::Intent)])), + AcquireOutcome::Ready ); + let entry = lm.table.get(&coll).unwrap(); + assert_eq!(entry.mode, LockMode::Intent); + assert_eq!(entry.holders.as_slice(), &[writer]); } #[test] fn younger_writer_waits() { let mut lm = LockManager::new(); - let t1 = txn(1, 1); // shared holder - let t2 = txn(1, 2); // exclusive requester, younger than t1 + let r = reservation(1, 1); + let writer = txn(2, 0); // younger than the reservation - assert_eq!(lm.acquire_shared(t1, key("k")), AcquireOutcome::Ready); - // The younger writer must not wound; it waits behind the shared holder. - assert_eq!(lm.acquire(t2, keyset(&["k"])), AcquireOutcome::Blocked); + assert_eq!(lm.acquire_shared(r, key("k")), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(writer, exclusive(&["k"])), + AcquireOutcome::Blocked + ); let entry = lm.table.get(&key("k")).unwrap(); - assert!(entry.holders.contains(&t1), "the shared holder still holds"); - assert!(!entry.holders.contains(&t2), "R holds nothing"); - assert!(entry.has_waiter(t2), "R is enqueued as an exclusive waiter"); + assert!(entry.holders.contains(&r), "the shared holder still holds"); + assert!(!entry.holders.contains(&writer)); + assert!(entry.has_waiter(writer)); } #[test] fn exclusive_waits_on_exclusive_regardless_of_age() { let mut lm = LockManager::new(); - let t2 = txn(1, 2); // exclusive holder (younger) - let t1 = txn(1, 1); // exclusive requester (older) + let t2 = txn(1, 2); + let t1 = txn(1, 1); - assert_eq!(lm.acquire(t2, keyset(&["k"])), AcquireOutcome::Ready); - // An exclusive holder is NEVER wounded, even by an older writer. - assert_eq!(lm.acquire(t1, keyset(&["k"])), AcquireOutcome::Blocked); + assert_eq!(lm.acquire(t2, exclusive(&["k"])), AcquireOutcome::Ready); + assert_eq!(lm.acquire(t1, exclusive(&["k"])), AcquireOutcome::Blocked); let entry = lm.table.get(&key("k")).unwrap(); - assert!( - entry.holders.contains(&t2), - "the exclusive holder is intact" - ); - assert!(!entry.holders.contains(&t1)); + assert_eq!(entry.holders.as_slice(), &[t2]); assert!(entry.has_waiter(t1)); } #[test] fn multi_key_atomic_wound_takes_both() { let mut lm = LockManager::new(); - let s1 = txn(1, 5); // shared holder on k1, younger than R - let s2 = txn(1, 6); // shared holder on k2, younger than R - let r = txn(1, 1); // exclusive requester, older than both + let s1 = reservation(1, 5); + let s2 = reservation(1, 6); + let r = txn(1, 1); assert_eq!(lm.acquire_shared(s1, key("k1")), AcquireOutcome::Ready); assert_eq!(lm.acquire_shared(s2, key("k2")), AcquireOutcome::Ready); - assert_eq!(lm.acquire(r, keyset(&["k1", "k2"])), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(r, exclusive(&["k1", "k2"])), + AcquireOutcome::Ready + ); for k in ["k1", "k2"] { let entry = lm.table.get(&key(k)).unwrap(); assert_eq!(entry.mode, LockMode::Exclusive); - assert!(entry.holders.contains(&r), "R holds {k}"); + assert_eq!(entry.holders.as_slice(), &[r]); } - assert!(!lm.table.get(&key("k1")).unwrap().holders.contains(&s1)); - assert!(!lm.table.get(&key("k2")).unwrap().holders.contains(&s2)); } #[test] - fn multi_key_atomic_wait_holds_none() { + fn multi_key_wait_takes_no_conflicting_key() { let mut lm = LockManager::new(); - let s1 = txn(1, 5); // shared holder on k1, younger than R - let s2 = txn(1, 0); // shared holder on k2, OLDER than R - let r = txn(1, 1); // exclusive requester + let s1 = reservation(1, 5); // younger than R + let s2 = reservation(0, 0); // older than R + let r = txn(1, 1); assert_eq!(lm.acquire_shared(s1, key("k1")), AcquireOutcome::Ready); assert_eq!(lm.acquire_shared(s2, key("k2")), AcquireOutcome::Ready); - // R is younger than the holder on k2, so it must wait on BOTH keys and - // hold neither (all-or-nothing). + // R is younger than the holder on k2, so it wounds nobody and waits on + // both keys behind their shared holders. assert_eq!( - lm.acquire(r, keyset(&["k1", "k2"])), + lm.acquire(r, exclusive(&["k1", "k2"])), AcquireOutcome::Blocked ); + for (k, holder) in [("k1", s1), ("k2", s2)] { + let entry = lm.table.get(&key(k)).unwrap(); + assert_eq!(entry.holders.as_slice(), &[holder]); + assert!(entry.has_waiter(r)); + } + } + + #[test] + fn self_upgrade_with_other_shared_holder_wounds_or_blocks() { + // The older reservation's self-upgrade wounds the younger one. + let mut lm = LockManager::new(); + let (r_old, r_young) = (reservation(1, 0), reservation(1, 1)); + + assert_eq!(lm.acquire_shared(r_old, key("k")), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(r_young, key("k")), AcquireOutcome::Ready); + assert_eq!(lm.acquire(r_old, exclusive(&["k"])), AcquireOutcome::Ready); + let entry = lm.table.get(&key("k")).unwrap(); + assert_eq!(entry.mode, LockMode::Exclusive); + assert_eq!(entry.holders.as_slice(), &[r_old]); - assert!( - !lm.table.get(&key("k1")).unwrap().holders.contains(&r), - "R holds no key" + // The younger one's self-upgrade drops its own hold and waits. + let mut lm = LockManager::new(); + assert_eq!(lm.acquire_shared(r_old, key("k")), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(r_young, key("k")), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(r_young, exclusive(&["k"])), + AcquireOutcome::Blocked ); - assert!(!lm.table.get(&key("k2")).unwrap().holders.contains(&r)); - // The older shared holder on k2 is untouched. - assert!(lm.table.get(&key("k2")).unwrap().holders.contains(&s2)); + let entry = lm.table.get(&key("k")).unwrap(); + assert_eq!(entry.mode, LockMode::Shared); + assert_eq!(entry.holders.as_slice(), &[r_old]); + assert!(entry.has_waiter(r_young)); + + assert_eq!(lm.release(r_old), vec![r_young]); + let entry = lm.table.get(&key("k")).unwrap(); + assert_eq!(entry.mode, LockMode::Exclusive); + assert_eq!(entry.holders.as_slice(), &[r_young]); } #[test] fn crossed_reservations_are_acyclic() { - // T1 holds shared K1 and wants exclusive K2; T2 holds shared K2 and - // wants exclusive K1. The older writer's exclusive acquire wounds the - // younger's shared holding, breaking the cycle — no deadlock. + // R1 reserves K1, R2 reserves K2. The older writer's exclusive acquire + // of K2 wounds the younger reservation, breaking any cycle. let mut lm = LockManager::new(); - let t1 = txn(1, 1); // older - let t2 = txn(1, 2); // younger - - assert_eq!(lm.acquire_shared(t1, key("k1")), AcquireOutcome::Ready); - assert_eq!(lm.acquire_shared(t2, key("k2")), AcquireOutcome::Ready); + let r1 = reservation(1, 1); + let r2 = reservation(1, 2); + let writer = txn(1, 0); - // T1 (older) acquires exclusive K2: wounds T2's shared holding and - // proceeds. - assert_eq!(lm.acquire(t1, keyset(&["k2"])), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(r1, key("k1")), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(r2, key("k2")), AcquireOutcome::Ready); + assert_eq!( + lm.acquire(writer, exclusive(&["k2"])), + AcquireOutcome::Ready + ); let k2 = lm.table.get(&key("k2")).unwrap(); assert_eq!(k2.mode, LockMode::Exclusive); - assert!(k2.holders.contains(&t1), "the older writer proceeds"); - assert!( - !k2.holders.contains(&t2), - "the younger's reservation is wounded away" - ); + assert_eq!(k2.holders.as_slice(), &[writer]); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/reap.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/reap.rs index b741bc756..a47ff6470 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/reap.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/reap.rs @@ -10,6 +10,8 @@ //! replicated logical threshold — no wall clock, so every replica reaps //! identically given the same input order. +use std::collections::BTreeSet; + use super::lock_entry::LockMode; use super::lock_key::TxnId; use super::manager::LockManager; @@ -22,23 +24,21 @@ impl LockManager { /// /// Restricted to owners in the reservation band (`is_reservation()`) — a real /// transaction's lock owner is never in this band, so a real txn's locks can - /// never be reaped. Further restricted to owners ALL of whose held keys are - /// still `Shared`: an owner that self-upgraded a key to `Exclusive` (a commit - /// in flight under that reservation) is left alone. + /// never be reaped. A reservation that still waits for a key is reaped too, + /// so it never blocks the waiters queued behind it. An owner that holds or + /// requests any key in a mode other than `Shared` (a commit in flight under + /// that reservation) is left alone. pub fn reap_expired_shared(&mut self, epoch_threshold: u64) -> Vec { - let expired: Vec = self + let owners: BTreeSet = self .held_locks - .iter() - .filter(|(owner, keys)| { - owner.is_reservation() - && owner.epoch < epoch_threshold - && keys.iter().all(|k| { - self.table - .get(k) - .is_some_and(|e| e.mode == LockMode::Shared) - }) - }) - .map(|(owner, _)| *owner) + .keys() + .chain(self.pending_keys.keys()) + .copied() + .filter(|owner| owner.is_reservation() && owner.epoch < epoch_threshold) + .collect(); + let expired: Vec = owners + .into_iter() + .filter(|owner| self.holds_only_shared(*owner)) .collect(); let mut promoted = Vec::new(); @@ -50,15 +50,34 @@ impl LockManager { ); promoted.extend(self.release(owner)); } + promoted.sort(); + promoted.dedup(); promoted } + + /// Whether every key `owner` holds, and every key it waits for, is + /// `Shared`. A key whose entry a wound revoked from `owner` does not count. + fn holds_only_shared(&self, owner: TxnId) -> bool { + let held_shared = self.held_locks.get(&owner).is_none_or(|keys| { + keys.iter().all(|key| { + self.table + .get(key) + .is_none_or(|e| !e.holders.contains(&owner) || e.mode == LockMode::Shared) + }) + }); + let waits_shared = self + .pending_keys + .get(&owner) + .is_none_or(|keys| keys.values().all(|mode| *mode == LockMode::Shared)); + held_shared && waits_shared + } } // ── Tests ───────────────────────────────────────────────────────────────────── #[cfg(test)] mod tests { - use std::collections::BTreeSet; + use std::collections::BTreeMap; use std::sync::Arc; use super::*; @@ -72,8 +91,11 @@ mod tests { } } - fn keyset(names: &[&str]) -> BTreeSet { - names.iter().map(|n| key(n)).collect() + fn keyset(names: &[&str]) -> BTreeMap { + names + .iter() + .map(|n| (key(n), LockMode::Exclusive)) + .collect() } fn txn(epoch: u64, pos: u32) -> TxnId { @@ -176,4 +198,20 @@ mod tests { assert_eq!(entry.holders.len(), 1); assert_eq!(entry.holders[0], writer); } + + #[test] + fn reap_releases_a_waiting_reservation() { + let mut lm = LockManager::new(); + let writer = txn(1, 0); + let r = resv(1, 0); + let later = txn(2, 0); + + assert_eq!(lm.acquire(writer, keyset(&["k"])), AcquireOutcome::Ready); + assert_eq!(lm.acquire_shared(r, key("k")), AcquireOutcome::Blocked); + assert_eq!(lm.acquire(later, keyset(&["k"])), AcquireOutcome::Blocked); + + assert!(lm.reap_expired_shared(100).is_empty()); + // The reaped reservation no longer queues ahead of `later`. + assert_eq!(lm.release(writer), vec![later]); + } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/metrics.rs b/nodedb/src/control/cluster/calvin/scheduler/metrics.rs index b87e85753..6371d5451 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/metrics.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/metrics.rs @@ -128,6 +128,7 @@ pub mod apply_halt_reason { pub const WAL_APPEND_FAILED: usize = 7; pub const METADATA_GROUP_GONE: usize = 8; pub const WRITE_VERSION_RECORD_FAILED: usize = 9; + pub const STAGE_VOTE_INVALID: usize = 10; pub const LABELS: &[&str] = &[ "draining", @@ -140,6 +141,7 @@ pub mod apply_halt_reason { "wal_append_failed", "metadata_group_gone", "write_version_record_failed", + "stage_vote_invalid", ]; } diff --git a/nodedb/src/control/crdt_admission.rs b/nodedb/src/control/crdt_admission.rs index c366b5e0b..05ba8e0cb 100644 --- a/nodedb/src/control/crdt_admission.rs +++ b/nodedb/src/control/crdt_admission.rs @@ -678,7 +678,7 @@ mod tests { payload: payload.into(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/gateway/error_map/system_dispatch_refusal.rs b/nodedb/src/control/gateway/error_map/system_dispatch_refusal.rs index f63dcf8b0..00cc42471 100644 --- a/nodedb/src/control/gateway/error_map/system_dispatch_refusal.rs +++ b/nodedb/src/control/gateway/error_map/system_dispatch_refusal.rs @@ -85,7 +85,7 @@ async fn refuse_once( payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: code.clone().map(Box::new), - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }, diff --git a/nodedb/src/control/gateway/outcome.rs b/nodedb/src/control/gateway/outcome.rs index 1a29714c8..965660db3 100644 --- a/nodedb/src/control/gateway/outcome.rs +++ b/nodedb/src/control/gateway/outcome.rs @@ -115,7 +115,7 @@ fn response( payload, watermark_lsn, error_code: error_code.map(Box::new), - read_set_valid: None, + stage_vote: None, read_version_lsn, write_set: Vec::new(), } diff --git a/nodedb/src/control/insert_select/orchestrator.rs b/nodedb/src/control/insert_select/orchestrator.rs index 0ea9ca185..a44dd75fa 100644 --- a/nodedb/src/control/insert_select/orchestrator.rs +++ b/nodedb/src/control/insert_select/orchestrator.rs @@ -222,7 +222,7 @@ pub(crate) async fn run_insert_select( payload: Payload::from_vec(payload), watermark_lsn: max_lsn, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), }) diff --git a/nodedb/src/control/local_dispatch/collect.rs b/nodedb/src/control/local_dispatch/collect.rs index 67ff384eb..909ec74db 100644 --- a/nodedb/src/control/local_dispatch/collect.rs +++ b/nodedb/src/control/local_dispatch/collect.rs @@ -169,7 +169,7 @@ mod collect_budget_tests { payload: Payload::from_vec(array_payload(n)), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } @@ -184,7 +184,7 @@ mod collect_budget_tests { payload: Payload::from_vec(array_payload(n)), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } @@ -200,7 +200,7 @@ mod collect_budget_tests { payload: Payload::from_vec(vec![0u8; bytes]), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } @@ -215,7 +215,7 @@ mod collect_budget_tests { payload: Payload::from_vec(vec![0u8; bytes]), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/local_dispatch/error_status.rs b/nodedb/src/control/local_dispatch/error_status.rs index b2fa8c0c6..4cd73e121 100644 --- a/nodedb/src/control/local_dispatch/error_status.rs +++ b/nodedb/src/control/local_dispatch/error_status.rs @@ -55,7 +55,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: Some(Box::new(code)), - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/orchestrated_write.rs b/nodedb/src/control/orchestrated_write.rs index e15400aa3..e890d48a5 100644 --- a/nodedb/src/control/orchestrated_write.rs +++ b/nodedb/src/control/orchestrated_write.rs @@ -59,7 +59,7 @@ pub(crate) async fn apply_orchestrated_write( payload: payload.into(), watermark_lsn: write_version, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: write_version, write_set: Vec::new(), }; @@ -80,7 +80,7 @@ fn data_plane_verdict(request_id: RequestId, code: ErrorCode) -> Response { payload: crate::bridge::envelope::Payload::from_vec(Vec::new()), watermark_lsn: crate::types::Lsn::ZERO, error_code: Some(Box::new(code)), - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/planner/calvin/submit/assign.rs b/nodedb/src/control/planner/calvin/submit/assign.rs index 4459347ad..4919bb63d 100644 --- a/nodedb/src/control/planner/calvin/submit/assign.rs +++ b/nodedb/src/control/planner/calvin/submit/assign.rs @@ -49,6 +49,7 @@ pub(crate) async fn submit_local_assign( ) -> crate::Result { super::local::raise_metadata_floor(state, &mut tx_class); super::local::stamp_incarnations(state, &mut tx_class)?; + super::unique_claims::stamp_unique_claims(state, &mut tx_class)?; let stream = super::parts::split_into_parts(state, &mut tx_class)?; assign_prepared(state, tx_class, stream, timeout).await } @@ -119,6 +120,7 @@ pub async fn submit_calvin_routed_assign( ) -> crate::Result { super::local::raise_metadata_floor(state, &mut tx_class); super::local::stamp_incarnations(state, &mut tx_class)?; + super::unique_claims::stamp_unique_claims(state, &mut tx_class)?; let stream = super::parts::split_into_parts(state, &mut tx_class)?; let local_timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); diff --git a/nodedb/src/control/planner/calvin/submit/local.rs b/nodedb/src/control/planner/calvin/submit/local.rs index 1916773cf..ead318c00 100644 --- a/nodedb/src/control/planner/calvin/submit/local.rs +++ b/nodedb/src/control/planner/calvin/submit/local.rs @@ -38,7 +38,7 @@ pub(super) fn synthetic_returning_response(payload_bytes: Vec) -> Response { payload: Payload::from_vec(payload_bytes), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } @@ -153,6 +153,7 @@ pub async fn submit_and_await_calvin_with_timeout( ) -> crate::Result> { raise_metadata_floor(state, &mut tx_class); stamp_incarnations(state, &mut tx_class)?; + super::unique_claims::stamp_unique_claims(state, &mut tx_class)?; let stream = super::parts::split_into_parts(state, &mut tx_class)?; submit_prepared_and_await(state, tx_class, stream, timeout).await } diff --git a/nodedb/src/control/planner/calvin/submit/mod.rs b/nodedb/src/control/planner/calvin/submit/mod.rs index 1deb2959d..ee43da39d 100644 --- a/nodedb/src/control/planner/calvin/submit/mod.rs +++ b/nodedb/src/control/planner/calvin/submit/mod.rs @@ -36,6 +36,7 @@ pub mod local; pub mod parts; pub mod routed; pub mod stream; +pub mod unique_claims; pub(crate) use assign::submit_local_assign; pub use assign::{RoutedAssignment, submit_calvin_routed_assign}; diff --git a/nodedb/src/control/planner/calvin/submit/routed.rs b/nodedb/src/control/planner/calvin/submit/routed.rs index 18ba17b47..07508790f 100644 --- a/nodedb/src/control/planner/calvin/submit/routed.rs +++ b/nodedb/src/control/planner/calvin/submit/routed.rs @@ -70,6 +70,7 @@ pub async fn submit_calvin_routed( ) -> crate::Result> { super::local::raise_metadata_floor(state, &mut tx_class); super::local::stamp_incarnations(state, &mut tx_class)?; + super::unique_claims::stamp_unique_claims(state, &mut tx_class)?; let stream = super::parts::split_into_parts(state, &mut tx_class)?; // Every running server has a cluster transport, the synthesized one-node // cluster included. Without one, `start_raft` never ran here. diff --git a/nodedb/src/control/planner/calvin/submit/unique_claims.rs b/nodedb/src/control/planner/calvin/submit/unique_claims.rs new file mode 100644 index 000000000..f10092dfd --- /dev/null +++ b/nodedb/src/control/planner/calvin/submit/unique_claims.rs @@ -0,0 +1,517 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! UNIQUE value claims of a Calvin transaction's writes. +//! +//! A document write that sets a UNIQUE-indexed value locks +//! `(collection, index, value)` exclusively. Two transactions that claim one +//! value then serialize, so the Data Plane's UNIQUE judgment of the second +//! one sees the first one's row on every replica. +//! +//! A write whose claimed value the plan does not carry (an expression +//! update, a body this module cannot decode) locks its whole collection +//! instead. That over-locks, and never under-locks. + +#![deny(clippy::wildcard_enum_match_arm)] + +use std::collections::{BTreeMap, BTreeSet}; + +use nodedb_cluster::calvin::types::{EngineKeySet, SortedVec, TxClass}; +use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan, UpdateValue}; + +use crate::Error; +use crate::control::state::SharedState; +use crate::engine::document::store::extract_index_values; + +/// One UNIQUE index of a collection, as its claims are extracted. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct UniquePath { + /// Index name. + pub name: String, + /// Indexed path, without a trailing `[]`. + pub path: String, + /// Whether the path indexes each array element. + pub is_array: bool, + /// Whether values are lowercased before the index judges them. + pub case_insensitive: bool, +} + +/// Add the UNIQUE claims of `tx_class`'s plans to its write set, read from +/// this node's catalog. +/// +/// A class already split into parts was stamped by its coordinator, so it is +/// left as it is. A forwarded class is stamped again with the same result: +/// a key set already in the write set is not added twice. +pub(crate) fn stamp_unique_claims( + state: &SharedState, + tx_class: &mut TxClass, +) -> crate::Result<()> { + if tx_class.is_multi_part() { + return Ok(()); + } + let plans = + nodedb_physical::physical_plan::wire::decode_batch(&tx_class.plans).map_err(|e| { + Error::Serialization { + format: "msgpack".into(), + detail: format!("calvin unique claims: plan decode: {e}"), + } + })?; + let catalog = state.credentials.catalog(); + let database_id = tx_class.database_id; + let tenant_id = tx_class.tenant_id.as_u64(); + let sets = unique_claim_sets(&plans, |collection| { + let key = nodedb_types::CollectionKey::from_qualified_str(database_id, collection) + .unwrap_or_else(|_| nodedb_types::CollectionKey::from_bare(database_id, collection)); + let stored = catalog.get_collection(key.database_id(), tenant_id, key.name())?; + Ok(stored.map_or_else(Vec::new, |stored| { + stored + .indexes + .iter() + .filter(|index| index.unique) + .map(|index| UniquePath { + name: index.name.clone(), + path: index + .field + .strip_suffix("[]") + .unwrap_or(&index.field) + .to_owned(), + is_array: index.field.ends_with("[]"), + case_insensitive: index.case_insensitive, + }) + .collect() + })) + })?; + for set in sets { + if !tx_class.write_set.0.contains(&set) { + tx_class.write_set.0.push(set); + } + } + Ok(()) +} + +/// The UNIQUE claim key sets of `plans`. `unique_paths` names the UNIQUE +/// indexes of a collection, as the plans name it. +pub(crate) fn unique_claim_sets( + plans: &[PhysicalPlan], + mut unique_paths: impl FnMut(&str) -> crate::Result>, +) -> crate::Result> { + let mut claims = Claims::default(); + let mut paths_of: BTreeMap> = BTreeMap::new(); + for plan in plans { + let op = match plan { + PhysicalPlan::Document(op) => op, + PhysicalPlan::Kv(_) + | PhysicalPlan::Vector(_) + | PhysicalPlan::Graph(_) + | PhysicalPlan::Crdt(_) + | PhysicalPlan::Columnar(_) + | PhysicalPlan::Timeseries(_) + | PhysicalPlan::Array(_) + | PhysicalPlan::Meta(_) + | PhysicalPlan::Text(_) + | PhysicalPlan::Spatial(_) + | PhysicalPlan::Query(_) + | PhysicalPlan::ClusterArray(_) + | PhysicalPlan::ClusterEvent(_) => continue, + }; + let Some(collection) = claiming_collection(op) else { + continue; + }; + if !paths_of.contains_key(collection) { + let paths = unique_paths(collection)?; + paths_of.insert(collection.to_owned(), paths); + } + let Some(paths) = paths_of.get(collection) else { + continue; + }; + if paths.is_empty() { + continue; + } + claims.add_op(collection, op, paths); + } + Ok(claims.into_key_sets()) +} + +/// The collection of a document op that can claim a UNIQUE value. `None` +/// for a delete, a read, a DDL op, a write that locks its collection whole, +/// and a write no Calvin transaction sequences. +fn claiming_collection(op: &DocumentOp) -> Option<&str> { + match op { + DocumentOp::PointPut { collection, .. } + | DocumentOp::PointInsert { collection, .. } + | DocumentOp::Upsert { collection, .. } + | DocumentOp::BatchInsert { collection, .. } + | DocumentOp::PointUpdate { collection, .. } + | DocumentOp::ApplyBalanceDelta { collection, .. } => Some(collection.as_str()), + DocumentOp::PointDelete { .. } + | DocumentOp::InsertSelect { .. } + | DocumentOp::BulkUpdate { .. } + | DocumentOp::BulkDelete { .. } + | DocumentOp::Truncate { .. } + | DocumentOp::Merge { .. } + | DocumentOp::UpdateFromJoin { .. } + | DocumentOp::ResolvedWrite { .. } + | DocumentOp::ResolveWrite(_) + | DocumentOp::PointGet { .. } + | DocumentOp::Scan { .. } + | DocumentOp::RangeScan { .. } + | DocumentOp::IndexLookup { .. } + | DocumentOp::IndexedFetch { .. } + | DocumentOp::EstimateCount { .. } + | DocumentOp::MaterializeScan { .. } + | DocumentOp::Register { .. } + | DocumentOp::DropIndex { .. } + | DocumentOp::BackfillIndex { .. } => None, + } +} + +/// Claimed values by collection and index, and collections locked whole +/// because a claim is unknown. Ordered maps keep the key sets identical on +/// every node. +#[derive(Default)] +struct Claims { + values: BTreeMap<(String, String), BTreeSet>>, + whole: BTreeSet, +} + +impl Claims { + /// Add the claims of `op` on `collection` against its UNIQUE `paths`. + fn add_op(&mut self, collection: &str, op: &DocumentOp, paths: &[UniquePath]) { + match op { + DocumentOp::PointPut { value, .. } | DocumentOp::PointInsert { value, .. } => { + self.add_body(collection, value, paths); + } + DocumentOp::Upsert { + value, + on_conflict_updates, + .. + } => { + self.add_body(collection, value, paths); + self.add_updates(collection, on_conflict_updates, paths); + } + DocumentOp::BatchInsert { documents, .. } => { + for (_, body) in documents { + self.add_body(collection, body, paths); + } + } + DocumentOp::PointUpdate { updates, .. } => { + self.add_updates(collection, updates, paths); + } + // A delta moves a numeric column by an amount the plan does not + // resolve to a value. + DocumentOp::ApplyBalanceDelta { column, .. } => { + if paths.iter().any(|path| touches(path, column)) { + self.whole.insert(collection.to_owned()); + } + } + DocumentOp::PointDelete { .. } + | DocumentOp::InsertSelect { .. } + | DocumentOp::BulkUpdate { .. } + | DocumentOp::BulkDelete { .. } + | DocumentOp::Truncate { .. } + | DocumentOp::Merge { .. } + | DocumentOp::UpdateFromJoin { .. } + | DocumentOp::ResolvedWrite { .. } + | DocumentOp::ResolveWrite(_) + | DocumentOp::PointGet { .. } + | DocumentOp::Scan { .. } + | DocumentOp::RangeScan { .. } + | DocumentOp::IndexLookup { .. } + | DocumentOp::IndexedFetch { .. } + | DocumentOp::EstimateCount { .. } + | DocumentOp::MaterializeScan { .. } + | DocumentOp::Register { .. } + | DocumentOp::DropIndex { .. } + | DocumentOp::BackfillIndex { .. } => {} + } + } + + /// Claim every UNIQUE value of a submitted document body. + fn add_body(&mut self, collection: &str, body: &[u8], paths: &[UniquePath]) { + let Some(doc) = decode_body(body) else { + self.whole.insert(collection.to_owned()); + return; + }; + for path in paths { + self.claim(collection, path, &doc); + } + } + + /// Claim the UNIQUE values field updates assign. An update that touches + /// an indexed path without a literal for exactly that path claims an + /// unknown value. + fn add_updates( + &mut self, + collection: &str, + updates: &[(String, UpdateValue)], + paths: &[UniquePath], + ) { + for (field, value) in updates { + for path in paths.iter().filter(|path| touches(path, field)) { + let literal = match value { + UpdateValue::Literal(bytes) if normalize(&path.path) == field.as_str() => { + nodedb_types::json_from_msgpack(bytes).ok() + } + UpdateValue::Literal(_) | UpdateValue::Expr(_) => None, + }; + match literal { + Some(literal) => { + let doc = nest(field, literal); + self.claim(collection, path, &doc); + } + None => { + self.whole.insert(collection.to_owned()); + } + } + } + } + } + + /// Claim the values of `path` in `doc`, canonicalized as the index + /// judges them. + fn claim(&mut self, collection: &str, path: &UniquePath, doc: &serde_json::Value) { + let values = extract_index_values(doc, &path.path, path.is_array); + if values.is_empty() { + return; + } + let claimed = self + .values + .entry((collection.to_owned(), path.name.clone())) + .or_default(); + for value in values { + let value = if path.case_insensitive { + value.to_lowercase() + } else { + value + }; + claimed.insert(value.into_bytes()); + } + } + + /// One `Unique` key set per collection and index, then one `Collection` + /// key set per collection with an unknown claim. + fn into_key_sets(self) -> Vec { + let mut sets: Vec = self + .values + .into_iter() + .map(|((collection, index), values)| EngineKeySet::Unique { + collection, + index, + values: SortedVec::new(values.into_iter().collect()), + }) + .collect(); + sets.extend( + self.whole + .into_iter() + .map(|collection| EngineKeySet::Collection { + collection, + vshards: SortedVec::new(Vec::new()), + }), + ); + sets + } +} + +/// `path` without a leading `$.` or `$`. +fn normalize(path: &str) -> &str { + path.strip_prefix("$.") + .or_else(|| path.strip_prefix('$')) + .unwrap_or(path) +} + +/// Whether assigning `field` can change the value at the indexed `path`: +/// the two name one field, or one lies inside the other. +fn touches(path: &UniquePath, field: &str) -> bool { + let indexed = normalize(&path.path); + indexed == field + || indexed + .strip_prefix(field) + .is_some_and(|rest| rest.starts_with('.')) + || field + .strip_prefix(indexed) + .is_some_and(|rest| rest.starts_with('.')) +} + +/// A document holding `value` at the dotted `field`. +fn nest(field: &str, value: serde_json::Value) -> serde_json::Value { + field.rsplit('.').fold(value, |inner, segment| { + let mut object = serde_json::Map::new(); + object.insert(segment.to_owned(), inner); + serde_json::Value::Object(object) + }) +} + +/// Decode a submitted document body: a MessagePack map or JSON. `None` for +/// any other form, such as a tagged value array or a strict Binary Tuple. +fn decode_body(body: &[u8]) -> Option { + let first = *body.first()?; + let msgpack_map = matches!(first, 0x80..=0x8f | 0xde | 0xdf); + let msgpack_array = matches!(first, 0x90..=0x9f | 0xdc | 0xdd); + if msgpack_map { + return nodedb_types::json_from_msgpack(body).ok(); + } + if msgpack_array { + return None; + } + sonic_rs::from_slice::(body) + .ok() + .filter(serde_json::Value::is_object) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::cluster::calvin::scheduler::{ + AcquireOutcome, LockKey, LockManager, LockMode, TxnId, + }; + use nodedb_types::{DatabaseId, QualifiedCollection, Surrogate}; + + const USERS: &str = "users"; + + fn email_index() -> UniquePath { + UniquePath { + name: "users_email".to_owned(), + path: "$.email".to_owned(), + is_array: false, + case_insensitive: true, + } + } + + fn paths(collection: &str) -> crate::Result> { + Ok(if collection == USERS { + vec![email_index()] + } else { + Vec::new() + }) + } + + fn insert(id: &str, surrogate: u32, email: &str) -> PhysicalPlan { + let body = nodedb_types::json_to_msgpack(&serde_json::json!({ "email": email })) + .expect("encode body"); + PhysicalPlan::Document(DocumentOp::PointInsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, USERS), + document_id: id.to_owned(), + value: body, + if_absent: false, + surrogate: Surrogate::new(surrogate), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: Vec::new(), + }) + } + + fn update(id: &str, value: UpdateValue) -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::PointUpdate { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, USERS), + document_id: id.to_owned(), + surrogate: Some(Surrogate::new(3)), + pk_bytes: id.as_bytes().to_vec(), + updates: vec![("email".to_owned(), value)], + returning: None, + rls_filters: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::pending_injection(), + resolved_sum_targets: Vec::new(), + declared_primary_key: None, + }) + } + + /// The scheduler's lock request for the claims of `plans`, as write keys. + fn claim_locks(plans: &[PhysicalPlan]) -> BTreeMap { + let sets = unique_claim_sets(plans, paths).expect("claims"); + let tx_class = TxClass::new_single_vshard( + nodedb_cluster::calvin::types::ReadWriteSet::new(Vec::new()), + nodedb_cluster::calvin::types::ReadWriteSet::new(sets), + Vec::new(), + nodedb_types::TenantId::new(1), + None, + nodedb_cluster::calvin::types::VersionedReadSet::default(), + ) + .expect("tx class"); + crate::control::cluster::calvin::scheduler::driver::helpers::expand_rw_set( + &nodedb_cluster::calvin::types::SequencedTxn { + epoch: 1, + position: 0, + tx_class, + epoch_system_ms: 1_700_000_000_000, + epoch_vshard_txn_count: 2, + lock_owner: None, + }, + ) + } + + #[test] + fn an_insert_claims_its_unique_value_canonicalized() { + let sets = unique_claim_sets(&[insert("a", 1, "A@x.io")], paths).expect("claims"); + assert_eq!( + sets, + vec![EngineKeySet::Unique { + collection: USERS.to_owned(), + index: "users_email".to_owned(), + values: SortedVec::new(vec![b"a@x.io".to_vec()]), + }] + ); + } + + #[test] + fn two_inserts_of_one_unique_value_serialize() { + let (first, second) = (TxnId::new(1, 0), TxnId::new(1, 1)); + let mut table = LockManager::new(); + assert_eq!( + table.acquire(first, claim_locks(&[insert("a", 1, "a@x.io")])), + AcquireOutcome::Ready + ); + assert_eq!( + table.acquire(second, claim_locks(&[insert("b", 2, "A@X.IO")])), + AcquireOutcome::Blocked, + "a second claim of one value waits for the first" + ); + let third = TxnId::new(1, 2); + assert_eq!( + table.acquire(third, claim_locks(&[insert("c", 4, "c@x.io")])), + AcquireOutcome::Ready, + "a claim of another value runs beside them" + ); + assert_eq!(table.release(first), vec![second]); + } + + #[test] + fn a_literal_update_claims_and_an_expression_update_locks_the_collection() { + let literal = nodedb_types::json_to_msgpack(&serde_json::json!("New@x.io")).expect("lit"); + let sets = + unique_claim_sets(&[update("a", UpdateValue::Literal(literal))], paths).expect("ok"); + assert_eq!( + sets, + vec![EngineKeySet::Unique { + collection: USERS.to_owned(), + index: "users_email".to_owned(), + values: SortedVec::new(vec![b"new@x.io".to_vec()]), + }] + ); + + let expr = UpdateValue::Expr(nodedb_query::expr::SqlExpr::Column("other".to_owned())); + let sets = unique_claim_sets(&[update("a", expr)], paths).expect("ok"); + assert_eq!( + sets, + vec![EngineKeySet::Collection { + collection: USERS.to_owned(), + vshards: SortedVec::new(Vec::new()), + }] + ); + } + + #[test] + fn a_collection_without_unique_indexes_claims_nothing() { + let plan = PhysicalPlan::Document(DocumentOp::PointInsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "plain"), + document_id: "a".to_owned(), + value: Vec::new(), + if_absent: false, + surrogate: Surrogate::new(1), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: Vec::new(), + }); + assert!(unique_claim_sets(&[plan], paths).expect("ok").is_empty()); + } +} diff --git a/nodedb/src/control/planner/calvin/tx_class/dependent_builder.rs b/nodedb/src/control/planner/calvin/tx_class/dependent_builder.rs index ad8130841..4b473fba6 100644 --- a/nodedb/src/control/planner/calvin/tx_class/dependent_builder.rs +++ b/nodedb/src/control/planner/calvin/tx_class/dependent_builder.rs @@ -21,7 +21,9 @@ use nodedb_physical::physical_task::PhysicalTask; use nodedb_types::{DatabaseId, TenantId}; use super::shared::{read_set_from, versioned_reads_from}; -use super::write_keys::task_write_keys; +use super::write_keys::{WriteKeys, add_plan_write_keys, task_write_keys}; +use crate::control::planner::calvin::is_dependent_predicate; +use crate::control::planner::calvin::write_class::is_write_plan; /// Build a **multi-vshard** `TxClass` for a dependent-read (OLLP) transaction. /// @@ -115,22 +117,23 @@ fn build_dependent_tx_class_impl( }); } + // A predicate that matched no rows, in a batch whose other tasks write + // nothing either, decides an empty statement: there is no state change to + // sequence, so no Calvin entry is proposed. The caller returns the + // statement's zero-row result instead of retrying. + if predicted_surrogates.is_empty() && !others_write(tasks, collection)? { + return Ok(None); + } + // Every other task's keys, extracted as `build_static_tx_class` extracts - // them. The OLLP collection's rows are the ones reconnaissance - // predicted, in place of the collection key its predicate write locks. + // them. The OLLP collection's rows are the ones reconnaissance predicted. + // Its predicate write keeps the whole-collection lock, so an insert that + // would join the predicate's match set waits for it. let mut keys = task_write_keys(tasks)?; keys.drop_documents(collection); keys.rows(collection, predicted_surrogates.iter().copied()); let write_set = ReadWriteSet::new(keys.into_key_sets()); - // A predicate that matched no rows, in a batch whose other tasks write - // nothing either, decides an empty write set: there is no state change to - // sequence, so no Calvin entry is proposed. `TxClass` rejects an empty - // write set, and that rejection is not a retryable condition — the caller - // returns the statement's zero-row result instead of retrying. - if write_set.is_empty() { - return Ok(None); - } // Populate the routing/identity read_set from the session read-set (a txn // that writes shard A but reads shard B enumerates B as a participant). An // empty `reads` slice yields an empty read_set. @@ -173,6 +176,20 @@ fn build_dependent_tx_class_impl( }) } +/// Whether any task besides the OLLP predicate write on `collection` writes +/// a key. +fn others_write(tasks: &[PhysicalTask], collection: &str) -> crate::Result { + let mut keys = WriteKeys::default(); + let others = tasks.iter().filter(|task| { + is_write_plan(&task.plan) + && !(is_dependent_predicate(&task.plan) && task.plan.collection() == Some(collection)) + }); + for task in others { + add_plan_write_keys(&mut keys, &task.plan)?; + } + Ok(keys.into_key_sets().iter().any(|set| !set.is_empty())) +} + #[cfg(test)] mod tests { use super::*; @@ -237,4 +254,78 @@ mod tests { .expect("zero-match predicate is not an error"); assert!(built.is_none(), "zero-match predicate builds no TxClass"); } + + /// The lock request the scheduler takes for `tx_class` at `position`. + fn locks( + tx_class: TxClass, + position: u32, + ) -> std::collections::BTreeMap< + crate::control::cluster::calvin::scheduler::LockKey, + crate::control::cluster::calvin::scheduler::LockMode, + > { + crate::control::cluster::calvin::scheduler::driver::helpers::expand_rw_set( + &nodedb_cluster::calvin::types::SequencedTxn { + epoch: 1, + position, + tx_class, + epoch_system_ms: 1_700_000_000_000, + epoch_vshard_txn_count: 2, + lock_owner: None, + }, + ) + } + + /// An OLLP predicate write locks its predicted rows and its whole + /// collection. An insert of a row the prediction never saw (a phantom) + /// waits for it, so the predicate cannot miss the row on one replica and + /// match it on another. + #[test] + fn an_ollp_predicate_write_blocks_a_phantom_insert() { + use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + use crate::control::cluster::calvin::scheduler::{AcquireOutcome, LockManager}; + + let predicate = build_single_vshard_dependent_tx_class( + &[bulk_delete_task("users")], + TenantId::new(1), + "users", + &[7], + &[], + ) + .expect("dependent TxClass") + .expect("non-empty write set"); + let insert_task = PhysicalTask { + tenant_id: TenantId::new(1), + vshard_id: VShardId::new(0), + database_id: DatabaseId::DEFAULT, + plan: PhysicalPlan::Document(DocumentOp::PointInsert { + collection: nodedb_types::QualifiedCollection::new(DatabaseId::DEFAULT, "users"), + document_id: "phantom".to_owned(), + value: Vec::new(), + if_absent: false, + surrogate: nodedb_types::Surrogate::new(99), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: Vec::new(), + }), + post_set_op: nodedb_physical::physical_task::PostSetOp::None, + txn_id: None, + }; + let insert = + super::super::build_single_vshard_tx_class(&[insert_task], TenantId::new(1), &[]) + .expect("insert TxClass"); + + let (first, second) = (TxnId::new(1, 0), TxnId::new(1, 1)); + let mut table = LockManager::new(); + assert_eq!( + table.acquire(first, locks(predicate, 0)), + AcquireOutcome::Ready + ); + assert_eq!( + table.acquire(second, locks(insert, 1)), + AcquireOutcome::Blocked, + "a phantom insert waits for the predicate write" + ); + assert_eq!(table.release(first), vec![second]); + } } diff --git a/nodedb/src/control/planner/calvin/tx_class/shared.rs b/nodedb/src/control/planner/calvin/tx_class/shared.rs index ea1f1f4ef..578c4d838 100644 --- a/nodedb/src/control/planner/calvin/tx_class/shared.rs +++ b/nodedb/src/control/planner/calvin/tx_class/shared.rs @@ -4,11 +4,14 @@ //! dependent `TxClass` builders. The write keys live in //! [`super::write_keys`]. +use std::collections::{BTreeMap, BTreeSet}; + use crate::control::server::shared::session::read_set::{ReadKey, ReadSetEntry}; use nodedb_cluster::calvin::types::{ EngineKeySet, EngineTag, ReadKeyIdent, ReadWriteSet, SortedVec, VersionedReadEntry, VersionedReadSet, }; +use nodedb_types::KeyRepr; /// Map the neutral session read-set into the replicated, LSN-versioned /// [`VersionedReadSet`] carried on the `TxClass`. @@ -46,90 +49,173 @@ pub(super) fn versioned_reads_from(reads: &[ReadSetEntry]) -> VersionedReadSet { ) } -/// Build the routing/identity `read_set` for a Calvin `TxClass` from the -/// neutral session read-set — the key-IDENTITY set for participant/routing, -/// not the LSN-versioned OCC set ([`versioned_reads_from`]). +/// Build the routing and lock `read_set` for a Calvin `TxClass` from the +/// neutral session read-set. This is the key-IDENTITY set for participants +/// and locks, not the LSN-versioned OCC set ([`versioned_reads_from`]). +/// +/// Each read maps to the keys the scheduler locks `Shared`: +/// - A point read locks its row: a surrogate, a KV key, or an edge's two +/// node lock pairs. +/// - A predicate, index-equality, or index-range read locks its whole +/// collection. /// -/// A [`ReadSetEntry`] carries only `(engine, collection)`, no key identity, -/// so an unhomed entry maps to a COLLECTION-homed [`EngineKeySet`] with an -/// empty key vector. That over-approximates participants: more validation, -/// never a dropped participant. A homed entry (one vShard of a cross-shard -/// graph read) maps to an `EngineKeySet::Edge` with no edges and its home in -/// `home_vshards`, so the vShard it read participates and validates it. +/// An unhomed read participates on its collection's vShard. A homed read +/// (one vShard of a cross-shard graph read) participates on its home only. +/// A homed read with no collection name enlists its home and locks nothing. pub(super) fn read_set_from(reads: &[ReadSetEntry]) -> ReadWriteSet { - use std::collections::BTreeSet; - - // Dedup by (engine-variant, collection): many reads on one collection - // collapse to a single keyset, keeping the Raft-log read_set compact. - // Vector/KV keep their engine variant; every other engine routes by - // collection name via a Document keyset. - let mut vector_colls: BTreeSet = BTreeSet::new(); - let mut kv_colls: BTreeSet = BTreeSet::new(); - let mut doc_colls: BTreeSet = BTreeSet::new(); - // Homed reads (cross-shard graph reads) participate on their home vShards, - // not on the collection's vShard. They key no edge, so they lock nothing. - let mut homed: std::collections::BTreeMap> = - std::collections::BTreeMap::new(); + let mut locks = ReadLocks::default(); for entry in reads { - if let Some(home) = entry.home { - homed - .entry(entry.collection.clone()) - .or_default() - .insert(home.as_u32()); - continue; + match entry.home { + Some(home) => locks.homed(entry, home.as_u32()), + None if entry.collection.is_empty() => {} + None => locks.unhomed(entry), } - if entry.collection.is_empty() { - continue; + } + ReadWriteSet::new(locks.into_key_sets()) +} + +/// The edge reads of one collection. +#[derive(Default)] +struct EdgeReads { + /// The node lock pairs the reads touch. + pairs: Vec<(u32, u32)>, + /// The vShards the collection is read on. + homes: Vec, +} + +/// The read lock keys of a session read-set, by engine and collection. +/// Every map is ordered, so the key sets are identical on every node. +#[derive(Default)] +struct ReadLocks { + documents: BTreeMap>, + vectors: BTreeMap>, + kv: BTreeMap>>, + edges: BTreeMap, + /// Collection to the vShards it is read whole on. Empty: its own vShard. + whole: BTreeMap>, +} + +impl ReadLocks { + /// A read homed on `home`: it participates there only. + fn homed(&mut self, entry: &ReadSetEntry, home: u32) { + let collection = entry.collection.clone(); + if collection.is_empty() { + self.edges.entry(collection).or_default().homes.push(home); + return; } - match entry.engine { - EngineTag::Vector => { - vector_colls.insert(entry.collection.clone()); - } - EngineTag::Kv => { - kv_colls.insert(entry.collection.clone()); + match &entry.key { + ReadKey::Point { repr } => { + let edges = self.edges.entry(collection).or_default(); + edges.homes.push(home); + match repr { + KeyRepr::Edge { src, dst, .. } => { + edges + .pairs + .extend([node_lock_pair(src), node_lock_pair(dst)]); + } + // A row key routes by its collection, not by this home, + // so a homed row read only enlists its home. + KeyRepr::Surrogate(_) | KeyRepr::KvKey(_) => {} + } } - EngineTag::Document - | EngineTag::Graph - | EngineTag::Text - | EngineTag::Columnar - | EngineTag::Timeseries - | EngineTag::Spatial - | EngineTag::Crdt - | EngineTag::Query - | EngineTag::Meta - | EngineTag::Array - | EngineTag::ClusterArray => { - doc_colls.insert(entry.collection.clone()); + ReadKey::Predicate | ReadKey::IndexEq { .. } | ReadKey::IndexRange { .. } => { + self.whole.entry(collection).or_default().insert(home); } } } - let mut sets: Vec = Vec::new(); - for collection in vector_colls { - sets.push(EngineKeySet::Vector { - collection, - surrogates: SortedVec::new(vec![]), - }); - } - for collection in kv_colls { - sets.push(EngineKeySet::Kv { - collection, - keys: SortedVec::new(vec![]), - }); - } - for collection in doc_colls { - sets.push(EngineKeySet::Document { - collection, - surrogates: SortedVec::new(vec![]), - }); + + /// A read of a named collection on the collection's own vShard. + fn unhomed(&mut self, entry: &ReadSetEntry) { + let collection = entry.collection.clone(); + match &entry.key { + ReadKey::Point { + repr: KeyRepr::Surrogate(surrogate), + } => { + let rows = match entry.engine { + EngineTag::Vector => &mut self.vectors, + EngineTag::Document + | EngineTag::Graph + | EngineTag::Kv + | EngineTag::Text + | EngineTag::Columnar + | EngineTag::Timeseries + | EngineTag::Spatial + | EngineTag::Crdt + | EngineTag::Query + | EngineTag::Meta + | EngineTag::Array + | EngineTag::ClusterArray => &mut self.documents, + }; + rows.entry(collection).or_default().push(*surrogate); + } + ReadKey::Point { + repr: KeyRepr::KvKey(key), + } => self.kv.entry(collection).or_default().push(key.to_vec()), + // An edge locks by node pairs, which route by key home. The + // collection's own vShard is enlisted by an empty document set. + ReadKey::Point { + repr: KeyRepr::Edge { src, dst, .. }, + } => { + self.edges + .entry(collection.clone()) + .or_default() + .pairs + .extend([node_lock_pair(src), node_lock_pair(dst)]); + self.documents.entry(collection).or_default(); + } + ReadKey::Predicate | ReadKey::IndexEq { .. } | ReadKey::IndexRange { .. } => { + self.whole.entry(collection).or_default(); + } + } } - for (collection, homes) in homed { - sets.push(EngineKeySet::Edge { - collection, - edges: SortedVec::new(vec![]), - home_vshards: SortedVec::new(homes.into_iter().collect()), - }); + + /// The key sets, one per engine and collection, ordered by collection. + fn into_key_sets(self) -> Vec { + let mut sets: Vec = Vec::new(); + sets.extend( + self.documents + .into_iter() + .map(|(collection, rows)| EngineKeySet::Document { + collection, + surrogates: SortedVec::new(rows), + }), + ); + sets.extend( + self.vectors + .into_iter() + .map(|(collection, rows)| EngineKeySet::Vector { + collection, + surrogates: SortedVec::new(rows), + }), + ); + sets.extend( + self.kv + .into_iter() + .map(|(collection, keys)| EngineKeySet::Kv { + collection, + keys: SortedVec::new(keys), + }), + ); + sets.extend( + self.edges + .into_iter() + .map(|(collection, reads)| EngineKeySet::Edge { + collection, + edges: SortedVec::new(reads.pairs), + home_vshards: SortedVec::new(reads.homes), + }), + ); + sets.extend( + self.whole + .into_iter() + .map(|(collection, vshards)| EngineKeySet::Collection { + collection, + vshards: SortedVec::new(vshards.into_iter().collect()), + }), + ); + sets.sort_by(|a, b| a.collection().cmp(b.collection())); + sets } - ReadWriteSet::new(sets) } /// The lock pair of node `node` within one edge collection. @@ -150,11 +236,12 @@ pub(crate) fn node_lock_pair(node: &str) -> (u32, u32) { #[cfg(test)] mod lockstep_tests { use super::*; - use crate::control::cluster::calvin::scheduler::lock_manager::LockKey; + use crate::control::cluster::calvin::scheduler::driver::helpers::expand_rw_set; + use crate::control::cluster::calvin::scheduler::lock_manager::{LockKey, LockMode}; use crate::control::planner::calvin::tx_class::static_builder::build_single_vshard_tx_class; use crate::control::server::shared::write_admission::lock_keys::plan_lock_keys; use crate::types::{DatabaseId, TenantId, VShardId}; - use nodedb_cluster::calvin::types::EngineKeySet; + use nodedb_cluster::calvin::types::{SequencedTxn, TxClass}; use nodedb_physical::physical_plan::{DocumentOp, GraphOp, KvOp, PhysicalPlan}; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; use nodedb_types::Surrogate; @@ -172,64 +259,22 @@ mod lockstep_tests { } } - /// Mirrors the scheduler's `expand_rw_set` `EngineKeySet` → `LockKey` - /// mapping (fixed translation). What's under test is whether the - /// extractor threads the same `(collection, key/surrogate)` as the - /// gate's `plan_lock_keys`, not this mapping. - fn scheduler_lock_keys(sets: &[EngineKeySet]) -> BTreeSet { - let mut keys = BTreeSet::new(); - for ks in sets { - match ks { - EngineKeySet::Document { - collection, - surrogates, - } - | EngineKeySet::Vector { - collection, - surrogates, - } => { - let coll: Arc = Arc::from(collection.as_str()); - for &surrogate in surrogates.iter() { - keys.insert(LockKey::Surrogate { - collection: Arc::clone(&coll), - surrogate, - }); - } - } - EngineKeySet::Kv { - collection, - keys: kv_keys, - } => { - let coll: Arc = Arc::from(collection.as_str()); - for k in kv_keys.iter() { - keys.insert(LockKey::Kv { - collection: Arc::clone(&coll), - key: Arc::from(k.as_slice()), - }); - } - } - EngineKeySet::Edge { - collection, edges, .. - } => { - let coll: Arc = Arc::from(collection.as_str()); - for &(src, dst) in edges.iter() { - keys.insert(LockKey::Edge { - collection: Arc::clone(&coll), - src, - dst, - }); - } - } - EngineKeySet::Array { collection, .. } => { - keys.insert(LockKey::Surrogate { - collection: Arc::from(collection.as_str()), - surrogate: - crate::control::planner::calvin::tx_class::write_keys::COLLECTION_KEY, - }); - } - } - } - keys + /// The row keys the scheduler locks `Exclusive` for `tx`, from its real + /// `expand_rw_set`. The collection's `Intent` key is left out: the gate + /// fences row keys only. + fn scheduler_lock_keys(tx: TxClass) -> BTreeSet { + expand_rw_set(&SequencedTxn { + epoch: 1, + position: 0, + tx_class: tx, + epoch_system_ms: 1_700_000_000_000, + epoch_vshard_txn_count: 1, + lock_owner: None, + }) + .into_iter() + .filter(|(_, mode)| *mode == LockMode::Exclusive) + .map(|(key, _)| key) + .collect() } fn assert_gate_matches_scheduler(plan: PhysicalPlan) { @@ -238,7 +283,7 @@ mod lockstep_tests { plan_lock_keys(&t.plan).expect("op must be fast-path eligible for this test"); let tx = build_single_vshard_tx_class(&[t], TenantId::new(1), &[]) .expect("valid single-vshard TxClass"); - let scheduler_keys = scheduler_lock_keys(&tx.write_set.0); + let scheduler_keys = scheduler_lock_keys(tx); assert_eq!( gate_keys, scheduler_keys, "gate and scheduler must lock the identical key set" diff --git a/nodedb/src/control/planner/calvin/tx_class/static_builder.rs b/nodedb/src/control/planner/calvin/tx_class/static_builder.rs index 6b18326de..c7f4c5b73 100644 --- a/nodedb/src/control/planner/calvin/tx_class/static_builder.rs +++ b/nodedb/src/control/planner/calvin/tx_class/static_builder.rs @@ -361,6 +361,88 @@ mod tests { } } + /// The lock request the scheduler takes for `tx_class` at `position`. + fn locks( + tx_class: TxClass, + position: u32, + ) -> std::collections::BTreeMap< + crate::control::cluster::calvin::scheduler::LockKey, + crate::control::cluster::calvin::scheduler::LockMode, + > { + crate::control::cluster::calvin::scheduler::driver::helpers::expand_rw_set( + &nodedb_cluster::calvin::types::SequencedTxn { + epoch: 1, + position, + tx_class, + epoch_system_ms: 1_700_000_000_000, + epoch_vshard_txn_count: 2, + lock_owner: None, + }, + ) + } + + /// A predicate read locks its whole collection `Shared`. A later row + /// writer of that collection takes `Intent` on it, so it waits until the + /// reader releases. A point read locks only its row. + #[test] + fn a_predicate_read_blocks_a_later_writer_until_released() { + use crate::control::cluster::calvin::scheduler::LockMode; + use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + use crate::control::cluster::calvin::scheduler::{AcquireOutcome, LockKey, LockManager}; + + let (col_a, col_b) = two_distinct_collections(); + let tasks = vec![point_insert_task(&col_a, 1), point_insert_task(&col_b, 2)]; + let reads = vec![ + read_entry("scan_col", ReadKey::Predicate, 11), + read_entry( + "read_col", + ReadKey::Point { + repr: KeyRepr::Surrogate(9), + }, + 10, + ), + ]; + let reader = build_static_tx_class(&tasks, TenantId::new(1), &reads) + .expect("valid multi-vShard TxClass"); + let reader_locks = locks(reader, 0); + assert_eq!( + reader_locks.get(&LockKey::Collection { + collection: std::sync::Arc::from("scan_col"), + }), + Some(&LockMode::Shared) + ); + assert_eq!( + reader_locks.get(&LockKey::Surrogate { + collection: std::sync::Arc::from("read_col"), + surrogate: 9, + }), + Some(&LockMode::Shared) + ); + assert!( + !reader_locks.contains_key(&LockKey::Collection { + collection: std::sync::Arc::from("read_col"), + }), + "a point read locks nothing collection-wide" + ); + + let writer = build_single_vshard_tx_class( + &[point_insert_task("scan_col", 5)], + TenantId::new(1), + &[], + ) + .expect("writer TxClass"); + + let (first, second) = (TxnId::new(1, 0), TxnId::new(1, 1)); + let mut table = LockManager::new(); + assert_eq!(table.acquire(first, reader_locks), AcquireOutcome::Ready); + assert_eq!( + table.acquire(second, locks(writer, 1)), + AcquireOutcome::Blocked, + "a row writer waits for the predicate reader of its collection" + ); + assert_eq!(table.release(first), vec![second]); + } + #[test] fn empty_read_set_yields_empty_read_and_versioned_reads() { let (col_a, col_b) = two_distinct_collections(); diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/columnar.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/columnar.rs index f815bdb73..5030ef1e5 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/columnar.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/columnar.rs @@ -12,8 +12,8 @@ use super::set::WriteKeys; /// Add the write keys of columnar op `op` to `keys`. pub(super) fn add_columnar_keys(keys: &mut WriteKeys, op: &ColumnarOp) -> crate::Result<()> { match op { - // Every row it inserts, and the collection key a multi-row write - // holds. + // Every row it inserts, and the whole collection, so inserts append + // to the collection in sequencer order. ColumnarOp::Insert { collection, surrogates, diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/document.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/document.rs index ebdd76bc1..ee6bd7c7f 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/document.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/document.rs @@ -43,10 +43,9 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &DocumentOp) -> crate::Result<( surrogate, .. } => keys.rows(collection.as_str(), [surrogate.as_u32()]), - // A key unbound in its database names no row yet: it takes the - // collection key and its row id. The row id orders it after an - // insert that binds the key, so its rebind at dispatch finds that - // binding on every replica. + // A key unbound in its database names no row yet: it locks only its + // row id. The row id orders it after an insert that binds the key, so + // its rebind at dispatch finds that binding on every replica. DocumentOp::PointDelete { collection, document_id, @@ -59,14 +58,12 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &DocumentOp) -> crate::Result<( surrogate, .. } => { - match surrogate { - Some(surrogate) => keys.rows(collection.as_str(), [surrogate.as_u32()]), - None => keys.whole_collection(collection.as_str()), + if let Some(surrogate) = surrogate { + keys.rows(collection.as_str(), [surrogate.as_u32()]); } keys.row_id(collection.as_str(), document_id); } - // Every row it inserts, by surrogate and row id, and the collection - // key a multi-row write holds. + // Every row it inserts, by surrogate and row id. DocumentOp::BatchInsert { collection, documents, @@ -77,7 +74,6 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &DocumentOp) -> crate::Result<( for (document_id, _) in documents { keys.row_id(collection.as_str(), document_id); } - keys.whole_collection(collection.as_str()); } DocumentOp::InsertSelect { target_collection, .. diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/graph.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/graph.rs index 2c66de50d..4b04c1095 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/graph.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/graph.rs @@ -64,9 +64,10 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &GraphOp) -> crate::Result<()> // whatever order this vShard flushes the edge writes in. GraphOp::TruncateEdges { collection, vshard } => keys.home(collection.as_str(), *vshard), // A CRDT delete's presence guard locks every document it names, - // stored or not, by the key every writer of that document locks, - // and the collection key every collection-wide write locks. So it - // runs after every lower writer of those documents flushed. + // stored or not, by the key every writer of that document locks. Its + // `Intent` on the collection orders it against every collection-wide + // write. So it runs after every lower writer of those documents + // flushed. GraphOp::NodePresenceGuard { collection, vshard, @@ -78,7 +79,6 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &GraphOp) -> crate::Result<()> collection.as_str(), present.iter().chain(absent.iter()).map(|id| row_id_key(id)), ); - keys.whole_collection(collection.as_str()); } GraphOp::ResolveEdgeDelete(_) | GraphOp::Hop { .. } diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/kv.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/kv.rs index 9e5974052..ff0227a25 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/kv.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/kv.rs @@ -61,16 +61,12 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &KvOp) -> crate::Result<()> { dest_key, .. } => keys.kv_keys(collection.as_str(), [source_key.clone(), dest_key.clone()]), - // Every key it writes, and the collection key a multi-row write - // holds. + // Every key it writes. KvOp::BatchPut { collection, entries, .. - } => { - keys.kv_keys(collection.as_str(), entries.iter().map(|(k, _)| k.clone())); - keys.whole_collection(collection.as_str()); - } + } => keys.kv_keys(collection.as_str(), entries.iter().map(|(k, _)| k.clone())), KvOp::Truncate { collection, .. } | KvOp::PredicateUpdate { collection, .. } | KvOp::PredicateDelete { collection, .. } => keys.whole_collection(collection.as_str()), diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/mod.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/mod.rs index 21df48d04..0d79c6654 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/mod.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/mod.rs @@ -14,4 +14,4 @@ pub mod set; pub mod vector; pub use plan::{add_plan_write_keys, task_write_keys}; -pub use set::{COLLECTION_KEY, WriteKeys, row_id_key}; +pub use set::{WriteKeys, row_id_key}; diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/plan.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/plan.rs index e02396acd..8ed05a490 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/plan.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/plan.rs @@ -24,9 +24,9 @@ pub fn task_write_keys(tasks: &[PhysicalTask]) -> crate::Result { /// Add the Calvin write keys of `plan` to `keys`. /// /// Every write op of every engine is matched by name. A write with row -/// identity locks its rows. A write without one locks -/// [`super::COLLECTION_KEY`] of its collection. A graph write locks its -/// edges and nodes on their key homes. +/// identity locks its rows, and the scheduler adds an `Intent` lock on their +/// collection. A write without one locks its whole collection. A graph write +/// locks its edges and nodes on their key homes. /// /// A write no single transaction can sequence, and a plan that writes /// nothing, is refused with a typed error. The caller never builds a @@ -75,7 +75,7 @@ pub(super) fn unsequenced(reason: &str) -> crate::Error { /// guard is sound only when the higher transaction waits for the lower one. #[cfg(test)] mod tests { - use std::collections::BTreeSet; + use std::collections::BTreeMap; use nodedb_cluster::calvin::types::SequencedTxn; use nodedb_physical::physical_plan::{ @@ -86,7 +86,9 @@ mod tests { use super::*; use crate::control::cluster::calvin::scheduler::driver::helpers::expand_rw_set; - use crate::control::cluster::calvin::scheduler::{AcquireOutcome, LockKey, LockManager, TxnId}; + use crate::control::cluster::calvin::scheduler::{ + AcquireOutcome, LockKey, LockManager, LockMode, TxnId, + }; use crate::control::planner::calvin::tx_class::build_single_vshard_tx_class; use crate::types::{DatabaseId, RecordHomes, TenantId, VShardId}; @@ -110,7 +112,7 @@ mod tests { } /// The lock keys the scheduler takes for a transaction of `plans`. - fn locks(plans: Vec, position: u32) -> BTreeSet { + fn locks(plans: Vec, position: u32) -> BTreeMap { let tasks: Vec = plans.into_iter().map(task).collect(); let tx_class = build_single_vshard_tx_class(&tasks, TenantId::new(1), &[]) .expect("a valid transaction class"); @@ -299,9 +301,9 @@ mod tests { /// An insert binds and stores `x` at a lower position. A delete or /// update planned while `x` was unbound must wait for it: its rebind at - /// dispatch then finds the binding on every replica. Before, it locked - /// only the collection key and can dispatch first on one replica and - /// second on another. + /// dispatch then finds the binding on every replica. Its row id lock + /// orders it, so it never dispatches first on one replica and second on + /// another. #[test] fn an_unbound_document_delete_or_update_waits_for_an_insert_of_its_id() { assert!(higher_waits( @@ -324,6 +326,45 @@ mod tests { assert!(higher_waits(vec![batch], vec![doc_delete("x", None)])); } + fn doc_truncate() -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::Truncate { + collection: qualified(DOCS), + restart_identity: false, + resolved_sum_targets: Vec::new(), + declared_primary_key: None, + }) + } + + /// A truncate locks its collection `Exclusive`, and a point write takes + /// `Intent` on it. So they serialize in sequencer order, whichever comes + /// first. + #[test] + fn a_truncate_and_a_point_write_serialize_in_sequence_order() { + assert!(higher_waits( + vec![doc_truncate()], + vec![doc_insert("x", 17)] + )); + assert!(higher_waits( + vec![doc_insert("x", 17)], + vec![doc_truncate()] + )); + assert!(higher_waits( + vec![doc_delete("x", None)], + vec![doc_truncate()] + )); + let collection_key = LockKey::Collection { + collection: std::sync::Arc::from(DOCS), + }; + assert_eq!( + locks(vec![doc_truncate()], 0).get(&collection_key), + Some(&LockMode::Exclusive) + ); + assert_eq!( + locks(vec![doc_insert("x", 17)], 0).get(&collection_key), + Some(&LockMode::Intent) + ); + } + /// Every point writer of one document id serializes, bound or not, and /// writers of distinct ids do not wait on each other. #[test] diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/set.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/set.rs index bed524f56..4e1125ae5 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/set.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/set.rs @@ -2,18 +2,13 @@ //! The accumulated Calvin write keys of a task slice. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use nodedb_cluster::calvin::types::{EngineKeySet, SortedVec}; use super::super::shared::node_lock_pair; use crate::types::{RecordHomes, VShardId}; -/// The row surrogate a write with no single row identity locks: a -/// predicate, bulk, truncate or multi-row write, or a point write whose key -/// is unbound. Every such write of a collection serializes on it. -pub const COLLECTION_KEY: u32 = 0; - /// The lock key of the row `document_id` names, bound or not: the bytes its /// surrogate binds under (`bind_plan`). /// @@ -43,6 +38,8 @@ pub struct WriteKeys { edges: BTreeMap, /// Array collection to the vShards its written cells live on. arrays: BTreeMap>, + /// Collections a write covers whole. + whole: BTreeSet, } impl WriteKeys { @@ -55,9 +52,11 @@ impl WriteKeys { .extend(surrogates); } - /// Lock [`COLLECTION_KEY`] of `collection`. + /// Lock `collection` whole, exclusively: a write with no row identity, + /// such as a predicate, bulk, or truncate write. It orders against every + /// row writer and every reader of the collection. pub fn whole_collection(&mut self, collection: &str) { - self.rows(collection, [COLLECTION_KEY]); + self.whole.insert(collection.to_owned()); } /// Lock the vector rows `surrogates` of `collection`. An empty list @@ -172,6 +171,14 @@ impl WriteKeys { vshards: SortedVec::new(vshards), }), ); + sets.extend( + self.whole + .into_iter() + .map(|collection| EngineKeySet::Collection { + collection, + vshards: SortedVec::new(Vec::new()), + }), + ); sets.sort_by(|a, b| a.collection().cmp(b.collection())); sets } diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/vector.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/vector.rs index e3fb7ccde..c3ea562f5 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/vector.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/vector.rs @@ -46,8 +46,8 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &VectorOp) -> crate::Result<()> document_surrogate, .. } => keys.vector_rows(collection.as_str(), [document_surrogate.as_u32()]), - // A key unbound in its database names no row: it takes the - // collection key. + // A key unbound in its database names no row: it locks the whole + // collection. VectorOp::DeleteBySurrogate { collection, surrogate, diff --git a/nodedb/src/control/planner/procedural/executor/core/dispatch.rs b/nodedb/src/control/planner/procedural/executor/core/dispatch.rs index 8759385a2..23b086141 100644 --- a/nodedb/src/control/planner/procedural/executor/core/dispatch.rs +++ b/nodedb/src/control/planner/procedural/executor/core/dispatch.rs @@ -494,7 +494,7 @@ mod origination_tests { payload: Payload::from_vec(payload), watermark_lsn: Lsn::ZERO, error_code, - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }; diff --git a/nodedb/src/control/request_tracker.rs b/nodedb/src/control/request_tracker.rs index 7fbafd085..cc7407edb 100644 --- a/nodedb/src/control/request_tracker.rs +++ b/nodedb/src/control/request_tracker.rs @@ -220,7 +220,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } @@ -235,7 +235,7 @@ mod tests { payload: Payload::from_vec(data.as_bytes().to_vec()), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/broadcast.rs b/nodedb/src/control/server/broadcast.rs index 06b7053f8..29e49c0d8 100644 --- a/nodedb/src/control/server/broadcast.rs +++ b/nodedb/src/control/server/broadcast.rs @@ -120,7 +120,7 @@ pub async fn broadcast_to_all_cores_txn( payload: crate::bridge::envelope::Payload::from_vec(outcome.merged_array), watermark_lsn: outcome.watermark_lsn, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), }) @@ -272,7 +272,7 @@ pub(crate) async fn broadcast_count_to_all_cores_until( payload: crate::bridge::envelope::Payload::from_vec(payload), watermark_lsn: max_lsn, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), }) diff --git a/nodedb/src/control/server/dispatch_utils/dispatch.rs b/nodedb/src/control/server/dispatch_utils/dispatch.rs index f6583661e..733b96742 100644 --- a/nodedb/src/control/server/dispatch_utils/dispatch.rs +++ b/nodedb/src/control/server/dispatch_utils/dispatch.rs @@ -572,7 +572,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }, @@ -751,7 +751,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: code.clone().map(Box::new), - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }, diff --git a/nodedb/src/control/server/dispatch_utils/durable_write.rs b/nodedb/src/control/server/dispatch_utils/durable_write.rs index b86295ec1..da5b51143 100644 --- a/nodedb/src/control/server/dispatch_utils/durable_write.rs +++ b/nodedb/src/control/server/dispatch_utils/durable_write.rs @@ -241,7 +241,7 @@ fn replicated_write_response( payload: payload.into(), watermark_lsn: write_version, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: write_version, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/dispatch_utils/minted/owned.rs b/nodedb/src/control/server/dispatch_utils/minted/owned.rs index fa40cf37f..5be212109 100644 --- a/nodedb/src/control/server/dispatch_utils/minted/owned.rs +++ b/nodedb/src/control/server/dispatch_utils/minted/owned.rs @@ -198,7 +198,7 @@ mod tests { constraint: "unique".into(), detail: "duplicate key".into(), })), - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/dispatch_utils/minted/resolve.rs b/nodedb/src/control/server/dispatch_utils/minted/resolve.rs index 32d400fcf..53c6baedf 100644 --- a/nodedb/src/control/server/dispatch_utils/minted/resolve.rs +++ b/nodedb/src/control/server/dispatch_utils/minted/resolve.rs @@ -121,7 +121,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/dispatch_utils/owner_read.rs b/nodedb/src/control/server/dispatch_utils/owner_read.rs index c14997490..d53847e7f 100644 --- a/nodedb/src/control/server/dispatch_utils/owner_read.rs +++ b/nodedb/src/control/server/dispatch_utils/owner_read.rs @@ -263,7 +263,7 @@ fn response( payload: Payload::from_vec(payload), watermark_lsn, error_code: error_code.map(Box::new), - read_set_valid: None, + stage_vote: None, read_version_lsn, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/exchange/all_cores/base_snapshot.rs b/nodedb/src/control/server/exchange/all_cores/base_snapshot.rs index 3c761b384..0fd10ea46 100644 --- a/nodedb/src/control/server/exchange/all_cores/base_snapshot.rs +++ b/nodedb/src/control/server/exchange/all_cores/base_snapshot.rs @@ -94,7 +94,7 @@ mod tests { payload: Payload::from_vec(payload.to_vec()), watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/exchange/core_outcome.rs b/nodedb/src/control/server/exchange/core_outcome.rs index 6b8746d42..5fed5382c 100644 --- a/nodedb/src/control/server/exchange/core_outcome.rs +++ b/nodedb/src/control/server/exchange/core_outcome.rs @@ -59,7 +59,7 @@ mod tests { payload: Payload::from_vec(payload.to_vec()), watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/exchange/response.rs b/nodedb/src/control/server/exchange/response.rs index 354357553..541836515 100644 --- a/nodedb/src/control/server/exchange/response.rs +++ b/nodedb/src/control/server/exchange/response.rs @@ -43,7 +43,7 @@ pub(super) fn outcome_to_response( payload: crate::bridge::envelope::Payload::from_vec(merged_array), watermark_lsn, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/graph_dispatch/helpers.rs b/nodedb/src/control/server/graph_dispatch/helpers.rs index 7aa3abbc5..1f63085ec 100644 --- a/nodedb/src/control/server/graph_dispatch/helpers.rs +++ b/nodedb/src/control/server/graph_dispatch/helpers.rs @@ -22,7 +22,7 @@ pub(super) fn ok_response(payload: Vec) -> Response { payload: crate::bridge::envelope::Payload::from_vec(payload), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/native/dispatch/raw_dispatch.rs b/nodedb/src/control/server/native/dispatch/raw_dispatch.rs index 298667222..d5c4e0ff3 100644 --- a/nodedb/src/control/server/native/dispatch/raw_dispatch.rs +++ b/nodedb/src/control/server/native/dispatch/raw_dispatch.rs @@ -178,7 +178,7 @@ async fn dispatch_external_crdt_apply( payload: Payload::from_vec(outcome.payload), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: outcome.write_version, write_set: Vec::new(), }) diff --git a/nodedb/src/control/server/native/dispatch/single_task.rs b/nodedb/src/control/server/native/dispatch/single_task.rs index d141355ee..250705625 100644 --- a/nodedb/src/control/server/native/dispatch/single_task.rs +++ b/nodedb/src/control/server/native/dispatch/single_task.rs @@ -122,7 +122,7 @@ pub(super) async fn dispatch_single_task( payload: Payload::from_vec(outcome.payload), watermark_lsn: Lsn::new(0), error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), }; diff --git a/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs b/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs index 72ebdd4d0..6741589f7 100644 --- a/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs +++ b/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs @@ -43,7 +43,7 @@ impl NodeDbPgHandler { // Authoritative participant WAL LSN — CDC ordering must use it, not zero. watermark_lsn: write_version, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: write_version, write_set: Vec::new(), }; diff --git a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs index 9dcb4878d..453266587 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs @@ -489,7 +489,7 @@ fn staged_rows_response(rows: Vec) -> crate::bridge::envelope::Response { payload: crate::bridge::envelope::Payload::from_vec(rows), watermark_lsn: crate::types::Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/result_stream.rs b/nodedb/src/control/server/result_stream.rs index 1f94856b9..28dcfef9e 100644 --- a/nodedb/src/control/server/result_stream.rs +++ b/nodedb/src/control/server/result_stream.rs @@ -156,7 +156,7 @@ mod tests { payload: Payload::from_vec(array_payload(n)), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } @@ -171,7 +171,7 @@ mod tests { payload: Payload::from_vec(array_payload(n)), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } @@ -186,7 +186,7 @@ mod tests { payload: Payload::from_vec(vec![0u8; bytes]), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } @@ -201,7 +201,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: Some(Box::new(code)), - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/shared/clone_read/dispatch.rs b/nodedb/src/control/server/shared/clone_read/dispatch.rs index 0ffc86b98..191729ef4 100644 --- a/nodedb/src/control/server/shared/clone_read/dispatch.rs +++ b/nodedb/src/control/server/shared/clone_read/dispatch.rs @@ -208,7 +208,7 @@ pub(super) fn empty_response(state: &SharedState) -> Response { payload: Payload::empty(), watermark_lsn: crate::types::Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/shared/clone_write/util.rs b/nodedb/src/control/server/shared/clone_write/util.rs index d55ee1f0c..208eb6b65 100644 --- a/nodedb/src/control/server/shared/clone_write/util.rs +++ b/nodedb/src/control/server/shared/clone_write/util.rs @@ -33,7 +33,7 @@ pub(super) fn synthetic_affected_response( payload: payload.into(), watermark_lsn, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs index b33617619..4d2f91726 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs @@ -155,7 +155,7 @@ fn verdict_response(code: ErrorCode) -> Response { payload: crate::bridge::envelope::Payload::empty(), watermark_lsn: crate::types::Lsn::ZERO, error_code: Some(Box::new(code)), - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/shared/ddl/neutral/rate_gate.rs b/nodedb/src/control/server/shared/ddl/neutral/rate_gate.rs index 7574d56e6..0690c931f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/rate_gate.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/rate_gate.rs @@ -431,7 +431,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs b/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs index 7a3a39111..4f992a398 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs @@ -340,7 +340,7 @@ mod tests { payload: payload.into(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn, write_set: Vec::new(), }, diff --git a/nodedb/src/control/server/shared/ddl/user_dispatch.rs b/nodedb/src/control/server/shared/ddl/user_dispatch.rs index c2a817eb2..1cf3bd18d 100644 --- a/nodedb/src/control/server/shared/ddl/user_dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/user_dispatch.rs @@ -492,7 +492,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }, diff --git a/nodedb/src/control/server/shared/metering.rs b/nodedb/src/control/server/shared/metering.rs index 21750b6d8..d0da235bc 100644 --- a/nodedb/src/control/server/shared/metering.rs +++ b/nodedb/src/control/server/shared/metering.rs @@ -962,7 +962,7 @@ mod tests { payload: crate::bridge::envelope::Payload::empty(), watermark_lsn: crate::types::Lsn::new(0), error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/shared/response_payload.rs b/nodedb/src/control/server/shared/response_payload.rs index 1c1f04a4b..53ce1feb5 100644 --- a/nodedb/src/control/server/shared/response_payload.rs +++ b/nodedb/src/control/server/shared/response_payload.rs @@ -44,7 +44,7 @@ mod tests { payload: Payload::from_vec(payload.to_vec()), watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/shared/session/lifecycle.rs b/nodedb/src/control/server/shared/session/lifecycle.rs index 2fddfda86..4094daebb 100644 --- a/nodedb/src/control/server/shared/session/lifecycle.rs +++ b/nodedb/src/control/server/shared/session/lifecycle.rs @@ -198,7 +198,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), }) diff --git a/nodedb/src/control/server/shared/session/read_set.rs b/nodedb/src/control/server/shared/session/read_set.rs index 0158b414f..392958c5a 100644 --- a/nodedb/src/control/server/shared/session/read_set.rs +++ b/nodedb/src/control/server/shared/session/read_set.rs @@ -376,6 +376,18 @@ fn lock_key_to_wire(key: &LockKey) -> LockKeyWire { src: *src, dst: *dst, }, + LockKey::Collection { collection } => LockKeyWire::Collection { + collection: collection.to_string(), + }, + LockKey::Unique { + collection, + index, + value, + } => LockKeyWire::Unique { + collection: collection.to_string(), + index: index.to_string(), + value: value.to_vec(), + }, } } diff --git a/nodedb/src/control/server/shared/session/savepoint_ops.rs b/nodedb/src/control/server/shared/session/savepoint_ops.rs index ad25f5f44..16a83560d 100644 --- a/nodedb/src/control/server/shared/session/savepoint_ops.rs +++ b/nodedb/src/control/server/shared/session/savepoint_ops.rs @@ -307,7 +307,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), }) diff --git a/nodedb/src/control/server/shared/write_admission/gate.rs b/nodedb/src/control/server/shared/write_admission/gate.rs index 16a5a6ea6..502a95c6c 100644 --- a/nodedb/src/control/server/shared/write_admission/gate.rs +++ b/nodedb/src/control/server/shared/write_admission/gate.rs @@ -36,7 +36,9 @@ use tokio::sync::mpsc::UnboundedSender; use crate::bridge::envelope::PhysicalPlan; use crate::control::cluster::calvin::scheduler::driver::core::routing::{PlanRouting, plan_vshard}; -use crate::control::cluster::calvin::scheduler::lock_manager::{LockKey, LockManager, TxnId}; +use crate::control::cluster::calvin::scheduler::lock_manager::{ + LockKey, LockManager, LockMode, TxnId, +}; use crate::control::planner::calvin::is_dependent_predicate; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId, VShardId}; @@ -249,7 +251,12 @@ pub fn admit(shared: &SharedState, target: &WriteTarget<'_>) -> WriteAdmission { ); let acquired = { let mut lm = lock_manager.lock().unwrap_or_else(|p| p.into_inner()); - lm.try_acquire(txn, keys) + lm.try_acquire( + txn, + keys.into_iter() + .map(|key| (key, LockMode::Exclusive)) + .collect(), + ) }; if acquired { // Look up this vShard's promotion channel so the guard can hand any diff --git a/nodedb/src/control/server/shared/write_admission/route.rs b/nodedb/src/control/server/shared/write_admission/route.rs index 2f63812ad..b3d9e2d9f 100644 --- a/nodedb/src/control/server/shared/write_admission/route.rs +++ b/nodedb/src/control/server/shared/write_admission/route.rs @@ -59,7 +59,7 @@ pub fn bare_ok_response(request_id: RequestId) -> Response { payload: crate::bridge::envelope::Payload::from_vec(Vec::new()), watermark_lsn: crate::types::Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/server/sync/raft_dispatch/response.rs b/nodedb/src/control/server/sync/raft_dispatch/response.rs index ed9f36a4d..36c16c3d0 100644 --- a/nodedb/src/control/server/sync/raft_dispatch/response.rs +++ b/nodedb/src/control/server/sync/raft_dispatch/response.rs @@ -72,7 +72,7 @@ fn ok_response(state: &SharedState, payload: Vec) -> Response { payload: payload.into(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/shutdown/data_plane.rs b/nodedb/src/control/shutdown/data_plane.rs index 812b49f04..a8b399046 100644 --- a/nodedb/src/control/shutdown/data_plane.rs +++ b/nodedb/src/control/shutdown/data_plane.rs @@ -287,7 +287,7 @@ mod tests { payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/state/calvin_ack_result.rs b/nodedb/src/control/state/calvin_ack_result.rs index 959ddb183..d50b74bec 100644 --- a/nodedb/src/control/state/calvin_ack_result.rs +++ b/nodedb/src/control/state/calvin_ack_result.rs @@ -164,7 +164,7 @@ mod tests { payload: crate::bridge::envelope::Payload::from_vec(payload), watermark_lsn: crate::types::Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/control/state/test_core.rs b/nodedb/src/control/state/test_core.rs index 04137eb56..bc0d95f5f 100644 --- a/nodedb/src/control/state/test_core.rs +++ b/nodedb/src/control/state/test_core.rs @@ -29,7 +29,7 @@ pub(crate) async fn acknowledge_every_request( payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }, diff --git a/nodedb/src/control/system_txn/tests.rs b/nodedb/src/control/system_txn/tests.rs index a4f061f60..f7e8962c9 100644 --- a/nodedb/src/control/system_txn/tests.rs +++ b/nodedb/src/control/system_txn/tests.rs @@ -111,7 +111,7 @@ async fn answer_all(state: Arc, mut side: CoreChannelDataSide, stop payload: Payload::empty(), watermark_lsn: Lsn::ZERO, error_code, - read_set_valid: None, + stage_vote: None, read_version_lsn: Lsn::ZERO, write_set: Vec::new(), }; diff --git a/nodedb/src/control/write_resolve/propose.rs b/nodedb/src/control/write_resolve/propose.rs index b8b1d371c..90a8eecad 100644 --- a/nodedb/src/control/write_resolve/propose.rs +++ b/nodedb/src/control/write_resolve/propose.rs @@ -54,7 +54,7 @@ pub(super) async fn propose_resolved( payload: payload.into(), watermark_lsn: write_version, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: write_version, write_set: Vec::new(), }; diff --git a/nodedb/src/data/executor/core_loop/response.rs b/nodedb/src/data/executor/core_loop/response.rs index c1aecc230..cc16de630 100644 --- a/nodedb/src/data/executor/core_loop/response.rs +++ b/nodedb/src/data/executor/core_loop/response.rs @@ -44,7 +44,7 @@ impl CoreLoop { watermark_lsn: self.watermark, read_version_lsn: self.read_version_lsn(task), error_code: None, - read_set_valid: None, + stage_vote: None, write_set: Vec::new(), } } @@ -63,7 +63,7 @@ impl CoreLoop { watermark_lsn: self.watermark, read_version_lsn: self.read_version_lsn(task), error_code: None, - read_set_valid: None, + stage_vote: None, write_set: Vec::new(), } } @@ -108,7 +108,7 @@ impl CoreLoop { watermark_lsn: self.watermark, read_version_lsn: self.read_version_lsn(task), error_code: None, - read_set_valid: None, + stage_vote: None, write_set: Vec::new(), } } @@ -165,7 +165,7 @@ impl CoreLoop { watermark_lsn: self.watermark, read_version_lsn: crate::types::Lsn::ZERO, error_code: Some(Box::new(error_code.into())), - read_set_valid: None, + stage_vote: None, write_set: Vec::new(), } } diff --git a/nodedb/src/data/executor/core_loop/tick.rs b/nodedb/src/data/executor/core_loop/tick.rs index e09a7a8c8..d894e9074 100644 --- a/nodedb/src/data/executor/core_loop/tick.rs +++ b/nodedb/src/data/executor/core_loop/tick.rs @@ -101,7 +101,7 @@ impl CoreLoop { watermark_lsn: self.watermark, // The task never started, so nothing it would write ran. error_code: Some(Box::new(ErrorCode::ExpiredBeforeExecution)), - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/data/executor/handlers/columnar_write/insert.rs b/nodedb/src/data/executor/handlers/columnar_write/insert.rs index dcd51cb13..c6d7f67bb 100644 --- a/nodedb/src/data/executor/handlers/columnar_write/insert.rs +++ b/nodedb/src/data/executor/handlers/columnar_write/insert.rs @@ -243,7 +243,7 @@ impl CoreLoop { payload: Payload::from_vec(json), watermark_lsn: self.watermark, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs b/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs index 3f07e7dc4..967c83fc9 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs @@ -10,7 +10,7 @@ use tracing::{debug, info_span}; use nodedb_cluster::calvin::types::PassiveReadKey; use nodedb_types::Value; -use crate::bridge::envelope::{ErrorCode, Payload, Response, Status}; +use crate::bridge::envelope::{ErrorCode, Payload, Response, StageVote, Status}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::commit_pending::PendingCommit; use crate::data::executor::handlers::control::calvin_reply::{CalvinReply, CalvinStaging}; @@ -142,8 +142,8 @@ impl CoreLoop { self.calvin.ollp_is_group_leader = prev_group_leader; match verified { Ok(true) => {} - Ok(false) => return self.response_error(task, ErrorCode::OllpRetryRequired), - Err(e) => return self.response_error(task, e), + Ok(false) => return self.calvin_stage_refusal(task, ErrorCode::OllpRetryRequired), + Err(e) => return self.calvin_stage_refusal(task, e.into()), } let synthetic_txn_id = match calvin_synthetic_txn_id(epoch, position, vshard_id) { @@ -203,9 +203,9 @@ impl CoreLoop { payload: Payload::empty(), watermark_lsn: self.watermark, error_code: None, - // The dependent-read path carries no versioned read-set; `None` maps - // to "commit" in `resolve_staged_commit` (`read_set_valid != Some(false)`). - read_set_valid: None, + // The dependent-read path carries no versioned read-set. Its OLLP + // check passed above, so the slice votes commit. + stage_vote: Some(StageVote::Commit), read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } @@ -249,9 +249,7 @@ mod tests { let resp = core.execute_calvin_execute_active(&task, ctx, &tenant_id, &plans, &injected); assert_eq!(resp.status, Status::Ok); - // The dependent-read path carries no versioned read-set; `None` maps to - // "commit" in `resolve_staged_commit`. - assert_eq!(resp.read_set_valid, None); + assert_eq!(resp.stage_vote, Some(StageVote::Commit)); let vshard_id = task.request.vshard_id.as_u32(); @@ -316,6 +314,7 @@ mod tests { resp.error_code.as_deref(), Some(&ErrorCode::OllpRetryRequired) ); + assert_eq!(resp.stage_vote, Some(StageVote::PredictionDrift)); // Drift stages NOTHING — neither the raw buffer nor the overlay. let vshard_id = task.request.vshard_id.as_u32(); @@ -329,4 +328,32 @@ mod tests { "an OLLP-drift retry must not leave a staged overlay" ); } + + #[test] + fn calvin_execute_active_stage_error_votes_participant_error() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let task = make_task(); + let tenant_id = TenantId::new(1); + let epoch_outside_synthetic_range = 1_u64 << 33; + let ctx = CalvinExecCtx { + epoch: epoch_outside_synthetic_range, + position: 0, + epoch_system_ms: 0, + is_group_leader: true, + }; + + let resp = core.execute_calvin_execute_active( + &task, + ctx, + &tenant_id, + &[point_insert_plan("orders", "o1", 7)], + &BTreeMap::new(), + ); + + assert_eq!(resp.status, Status::Error); + assert_eq!(resp.stage_vote, Some(StageVote::ParticipantError)); + assert!(core.calvin.commit_pending.is_empty()); + assert!(core.txn_overlays.is_empty()); + } } diff --git a/nodedb/src/data/executor/handlers/control/calvin/shared.rs b/nodedb/src/data/executor/handlers/control/calvin/shared.rs index bbabcade5..d690594b4 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/shared.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/shared.rs @@ -3,7 +3,7 @@ //! `CalvinExecCtx`, the helpers the static and active Calvin stage paths //! share, and the test fixtures every concern file's tests build on. -use crate::bridge::envelope::{ErrorCode, Response}; +use crate::bridge::envelope::{ErrorCode, Response, StageVote}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::task::ExecutionTask; @@ -20,7 +20,18 @@ pub(in crate::data::executor) struct CalvinExecCtx { } impl CoreLoop { - /// Clean failed static staging and return an explicit abort vote. + /// Refuse a Calvin stage that staged nothing, with the vote `error` names. + /// + /// `OllpRetryRequired` votes `PredictionDrift`. Every other error votes + /// `ParticipantError`. + pub(super) fn calvin_stage_refusal(&self, task: &ExecutionTask, error: ErrorCode) -> Response { + let vote = StageVote::of_stage_error(&error); + let mut response = self.response_error(task, error); + response.stage_vote = Some(vote); + response + } + + /// Clean failed staging and return its abort vote. /// Defensive removal clears document/KV and graph overlays plus their gauge. pub(super) fn calvin_stage_failure( &mut self, @@ -40,11 +51,9 @@ impl CoreLoop { self.calvin .fence .note_resolved((epoch, position, vshard_id), None); - let mut response = self.response_error(task, error.into()); - // Scheduler treats this as a durable local abort vote and still waits - // for the authoritative global verdict before issuing any drop. - response.read_set_valid = Some(false); - response + // The scheduler proposes this abort vote and waits for the global + // verdict before it drops anything. + self.calvin_stage_refusal(task, error.into()) } } @@ -174,3 +183,48 @@ pub(in crate::data::executor::handlers::control::calvin) mod test_support { .expect("seed row"); } } + +#[cfg(test)] +mod tests { + use super::test_support::make_task; + use crate::bridge::envelope::{ErrorCode, StageVote, Status}; + use crate::data::executor::core_loop::tests::make_core_with_dir; + use crate::data::executor::handlers::control::calvin_txn_id::calvin_synthetic_txn_id; + + #[test] + fn a_stage_drift_refusal_votes_prediction_drift_and_drops_staged_state() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let task = make_task(); + let vshard = task.request.vshard_id.as_u32(); + let synthetic = calvin_synthetic_txn_id(4, 2, vshard).unwrap(); + core.graph_txn_overlay_mut(synthetic); + + let response = core.calvin_stage_failure(&task, 4, 2, vshard, ErrorCode::OllpRetryRequired); + + assert_eq!(response.status, Status::Error); + assert_eq!(response.stage_vote, Some(StageVote::PredictionDrift)); + assert_eq!( + response.error_code.as_deref(), + Some(&ErrorCode::OllpRetryRequired) + ); + assert!(!core.graph_txn_overlays.contains_key(&synthetic)); + } + + #[test] + fn any_other_stage_refusal_votes_participant_error() { + let dir = tempfile::tempdir().unwrap(); + let (core, _tx, _rx) = make_core_with_dir(dir.path()); + let task = make_task(); + + let response = core.calvin_stage_refusal( + &task, + ErrorCode::Internal { + detail: "stage".into(), + }, + ); + + assert_eq!(response.status, Status::Error); + assert_eq!(response.stage_vote, Some(StageVote::ParticipantError)); + } +} diff --git a/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs b/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs index 8fc7ba688..6e9d44ff2 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs @@ -8,7 +8,7 @@ use tracing::{debug, info_span}; use nodedb_types::calvin::VersionedReadEntry; -use crate::bridge::envelope::{ErrorCode, Payload, Response, Status}; +use crate::bridge::envelope::{ErrorCode, Payload, Response, StageVote, Status}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::commit_pending::PendingCommit; use crate::data::executor::handlers::control::calvin_reply::{CalvinReply, CalvinStaging}; @@ -32,7 +32,7 @@ impl CoreLoop { /// effects — nothing is observable until a subsequent /// [`CoreLoop::execute_calvin_flush`] installs the transaction's redo /// record (or [`CoreLoop::execute_calvin_drop`] discards the staged - /// state). The response carries the vote on `read_set_valid`. Staging runs + /// state). The response carries the vote on `stage_vote`. Staging runs /// under the epoch's deterministic time anchor, which the pending entry /// keeps for resolve. `body_plans` indexes the plans a trigger body /// buffered: they stage under `Trigger`, so their rows fire no trigger. @@ -125,7 +125,11 @@ impl CoreLoop { // current against the local write versions? Empty read-set is vacuously // current. Read-only — no base mutation here. A stale-read false vote // retains the fully staged state until the durable global verdict. - let vote = self.read_set_still_current(task, tenant_id.as_u64(), versioned_reads); + let vote = StageVote::of_read_set(self.read_set_still_current( + task, + tenant_id.as_u64(), + versioned_reads, + )); // Publish only a fully staged transaction. Resolve reads these plans // against the overlay; a global abort drops both. @@ -147,7 +151,7 @@ impl CoreLoop { payload: Payload::empty(), watermark_lsn: self.watermark, error_code: None, - read_set_valid: Some(vote), + stage_vote: Some(vote), read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } @@ -216,6 +220,8 @@ mod tests { let resp = core.execute_calvin_execute_static(&task, ctx, &tenant_id, &plans, &[], &[]); assert_eq!(resp.status, Status::Ok); + // An empty read-set is current. + assert_eq!(resp.stage_vote, Some(StageVote::Commit)); let vshard_id = task.request.vshard_id.as_u32(); @@ -239,6 +245,87 @@ mod tests { ); } + /// A predicate read of `collection` on vShard 0, taken at `read_lsn`. + fn homed_read(collection: &str, read_lsn: u64) -> VersionedReadEntry { + VersionedReadEntry { + engine: nodedb_types::calvin::EngineTag::Document, + collection: collection.to_string(), + key: nodedb_types::calvin::ReadKeyIdent::Predicate, + read_lsn: crate::types::Lsn::new(read_lsn), + home_vshard: Some(0), + served_by: 0, + } + } + + #[test] + fn static_stage_with_a_superseded_read_votes_serialization_conflict() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + core.note_write_lsn( + DatabaseId::DEFAULT, + TenantId::new(1), + "orders", + None, + crate::types::Lsn::new(20), + ); + let task = make_task(); + let tenant_id = TenantId::new(1); + let ctx = CalvinExecCtx { + epoch: 3, + position: 0, + epoch_system_ms: 0, + is_group_leader: true, + }; + + let resp = core.execute_calvin_execute_static( + &task, + ctx, + &tenant_id, + &[point_insert_plan("orders", "o1", 7)], + &[homed_read("orders", 10)], + &[], + ); + + assert_eq!(resp.status, Status::Ok); + assert_eq!(resp.stage_vote, Some(StageVote::SerializationConflict)); + // A stale slice stays staged until the global verdict drops it. + let vshard = task.request.vshard_id.as_u32(); + assert!(core.calvin.commit_pending.contains_key(&(3, 0, vshard))); + } + + #[test] + fn static_stage_with_a_current_read_votes_commit() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + core.note_write_lsn( + DatabaseId::DEFAULT, + TenantId::new(1), + "orders", + None, + crate::types::Lsn::new(20), + ); + let task = make_task(); + let tenant_id = TenantId::new(1); + let ctx = CalvinExecCtx { + epoch: 3, + position: 1, + epoch_system_ms: 0, + is_group_leader: true, + }; + + let resp = core.execute_calvin_execute_static( + &task, + ctx, + &tenant_id, + &[point_insert_plan("orders", "o1", 7)], + &[homed_read("orders", 20)], + &[], + ); + + assert_eq!(resp.status, Status::Ok); + assert_eq!(resp.stage_vote, Some(StageVote::Commit)); + } + #[test] fn synthetic_id_failure_leaves_no_pending_or_overlay_state() { let dir = tempfile::tempdir().unwrap(); @@ -262,7 +349,7 @@ mod tests { ); assert_eq!(response.status, Status::Error); - assert_eq!(response.read_set_valid, Some(false)); + assert_eq!(response.stage_vote, Some(StageVote::ParticipantError)); assert!(core.calvin.commit_pending.is_empty()); assert!(core.txn_overlays.is_empty()); assert!(core.graph_txn_overlays.is_empty()); @@ -306,7 +393,7 @@ mod tests { let response = core.execute_calvin_execute_static(&task, ctx, &tenant_id, &plans, &[], &[]); assert_eq!(response.status, Status::Error); - assert_eq!(response.read_set_valid, Some(false)); + assert_eq!(response.stage_vote, Some(StageVote::ParticipantError)); assert!(!core.calvin.commit_pending.contains_key(&(9, 3, vshard))); assert!(!core.txn_overlays.contains_key(&synthetic)); assert!(!core.graph_txn_overlays.contains_key(&synthetic)); @@ -343,7 +430,7 @@ mod tests { &[], ); - assert_eq!(response.read_set_valid, Some(false)); + assert_eq!(response.stage_vote, Some(StageVote::ParticipantError)); assert!(!core.graph_txn_overlays.contains_key(&synthetic)); } @@ -391,7 +478,7 @@ mod tests { ); assert_eq!(response.status, Status::Error); - assert_eq!(response.read_set_valid, Some(false)); + assert_eq!(response.stage_vote, Some(StageVote::ParticipantError)); assert!(!core.calvin.commit_pending.contains_key(&(9, 4, vshard))); assert!(!core.txn_overlays.contains_key(&synthetic)); assert!(!core.graph_txn_overlays.contains_key(&synthetic)); @@ -435,7 +522,7 @@ mod tests { &[], &[], ); - assert_eq!(failed.read_set_valid, Some(false)); + assert_eq!(failed.stage_vote, Some(StageVote::ParticipantError)); assert!(matches!( failed.error_code.as_deref(), Some(ErrorCode::RejectedPrevalidation { .. }) @@ -457,7 +544,7 @@ mod tests { &[], &[], ); - assert_eq!(staged.read_set_valid, Some(true)); + assert_eq!(staged.stage_vote, Some(StageVote::Commit)); assert_eq!( core.txn_overlays .get(&synthetic) @@ -480,7 +567,7 @@ mod tests { &[], ); let mismatch_id = calvin_synthetic_txn_id(21, 2, vshard).expect("synthetic transaction id"); - assert_eq!(failed.read_set_valid, Some(false)); + assert_eq!(failed.stage_vote, Some(StageVote::ParticipantError)); assert!(!core.calvin.commit_pending.contains_key(&(21, 2, vshard))); assert!(!core.txn_overlays.contains_key(&mismatch_id)); @@ -505,7 +592,7 @@ mod tests { ); let overflow_id = calvin_synthetic_txn_id(21, 3, vshard).expect("synthetic transaction id"); assert_eq!(failed.status, Status::Error); - assert_eq!(failed.read_set_valid, Some(false)); + assert_eq!(failed.stage_vote, Some(StageVote::ParticipantError)); assert!(matches!( failed.error_code.as_deref(), Some(ErrorCode::RejectedPrevalidation { .. }) diff --git a/nodedb/src/data/executor/handlers/control/calvin_passive_read.rs b/nodedb/src/data/executor/handlers/control/calvin_passive_read.rs index 1e6f76929..1fbb62cbb 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_passive_read.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_passive_read.rs @@ -107,6 +107,9 @@ impl CoreLoop { // An array write is a static write: no dependent transaction // predicts its cells, so no passive participant reads them. EngineKeySet::Array { .. } => Vec::new(), + // A whole-collection key and a UNIQUE value key are lock keys + // only: no dependent transaction predicts them. + EngineKeySet::Collection { .. } | EngineKeySet::Unique { .. } => Vec::new(), } } diff --git a/nodedb/src/data/executor/handlers/query_collection_size.rs b/nodedb/src/data/executor/handlers/query_collection_size.rs index 5f4d1d76c..6916001ba 100644 --- a/nodedb/src/data/executor/handlers/query_collection_size.rs +++ b/nodedb/src/data/executor/handlers/query_collection_size.rs @@ -79,7 +79,7 @@ impl CoreLoop { payload: Payload::from(payload), watermark_lsn: self.watermark, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest.rs b/nodedb/src/data/executor/handlers/timeseries/ingest.rs index 9771b5586..875da439b 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest.rs @@ -431,7 +431,7 @@ impl CoreLoop { payload: Payload::from_vec(json), watermark_lsn: self.watermark, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest_dispatch.rs b/nodedb/src/data/executor/handlers/timeseries/ingest_dispatch.rs index 46888bca3..f10ac9ab3 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest_dispatch.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest_dispatch.rs @@ -173,7 +173,7 @@ impl CoreLoop { payload: Payload::from_vec(json), watermark_lsn: self.watermark, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), }; diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest_resolved.rs b/nodedb/src/data/executor/handlers/timeseries/ingest_resolved.rs index f2c387510..7839ce88b 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest_resolved.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest_resolved.rs @@ -382,7 +382,7 @@ impl CoreLoop { payload: Payload::from_vec(json), watermark_lsn: self.watermark, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), }, diff --git a/nodedb/src/data/executor/handlers/timeseries/scan.rs b/nodedb/src/data/executor/handlers/timeseries/scan.rs index f986f9b46..35475a182 100644 --- a/nodedb/src/data/executor/handlers/timeseries/scan.rs +++ b/nodedb/src/data/executor/handlers/timeseries/scan.rs @@ -274,7 +274,7 @@ impl CoreLoop { payload: Payload::from_vec(buf), watermark_lsn: self.watermark, error_code: None, - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), } diff --git a/nodedb/src/data/runtime/event_loop.rs b/nodedb/src/data/runtime/event_loop.rs index 08ba80cc5..5fbba58f2 100644 --- a/nodedb/src/data/runtime/event_loop.rs +++ b/nodedb/src/data/runtime/event_loop.rs @@ -201,7 +201,7 @@ fn drain_and_reject(core: &mut CoreLoop, core_id: usize) { error_code: Some(Box::new(ErrorCode::Internal { detail: format!("core-{core_id} is degraded after repeated panics"), })), - read_set_valid: None, + stage_vote: None, read_version_lsn: crate::types::Lsn::ZERO, write_set: Vec::new(), }; diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply.rs index c65106ffd..3fc15f168 100644 --- a/nodedb/tests/inproc/cases/calvin_two_phase_apply.rs +++ b/nodedb/tests/inproc/cases/calvin_two_phase_apply.rs @@ -3,7 +3,7 @@ //! Staged Calvin apply on the Data Plane: `MetaOp::CalvinExecuteStatic` //! VALIDATES + STAGES a transaction's write plans into the commit-pending //! buffer WITHOUT mutating base, returning the local commit vote on -//! `read_set_valid`. `MetaOp::CalvinResolve` then resolves the staged plans +//! `stage_vote`. `MetaOp::CalvinResolve` then resolves the staged plans //! into the transaction's redo record, and `MetaOp::CalvinFlush` installs it //! at its LSN (making the write visible), or `MetaOp::CalvinDrop` discards //! the staged state (leaving base unchanged). @@ -16,7 +16,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use nodedb::bridge::dispatch::{BridgeRequest, BridgeResponse}; -use nodedb::bridge::envelope::{Priority, Request, Response, Status}; +use nodedb::bridge::envelope::{Priority, Request, Response, StageVote, Status}; use nodedb::data::executor::core_loop::CoreLoop; use nodedb::types::*; use nodedb_bridge::buffer::{Consumer, Producer, RingBuffer}; @@ -264,8 +264,8 @@ fn flush_makes_staged_calvin_write_visible() { ); assert_eq!(staged.status, Status::Ok, "stage must succeed"); assert_eq!( - staged.read_set_valid, - Some(true), + staged.stage_vote, + Some(StageVote::Commit), "empty read-set is vacuously current → commit vote" ); @@ -373,8 +373,8 @@ fn drop_discards_invalid_staged_calvin_write() { "stage must succeed even on abort vote" ); assert_eq!( - staged.read_set_valid, - Some(false), + staged.stage_vote, + Some(StageVote::SerializationConflict), "stale read-set must produce an abort vote" ); @@ -474,8 +474,8 @@ fn point_read_at_write_lsn_commits_and_flush_applies() { ); assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); assert_eq!( - staged.read_set_valid, - Some(true), + staged.stage_vote, + Some(StageVote::Commit), "a read at or after the last write LSN must be current -> commit vote" ); @@ -563,8 +563,8 @@ fn stale_point_read_of_kv_key_aborts_stage_and_drop_discards() { "stage must succeed even on abort vote" ); assert_eq!( - staged.read_set_valid, - Some(false), + staged.stage_vote, + Some(StageVote::SerializationConflict), "a stale Point read (write after read_lsn) must produce an abort vote" ); @@ -677,8 +677,8 @@ fn absent_kv_key_phantom_insert_causes_abort() { ); assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); assert_eq!( - staged.read_set_valid, - Some(false), + staged.stage_vote, + Some(StageVote::SerializationConflict), "a phantom insert into a key observed absent must abort the stage vote" ); } @@ -757,8 +757,8 @@ fn absent_document_phantom_insert_is_caught() { ); assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); assert_eq!( - staged.read_set_valid, - Some(false), + staged.stage_vote, + Some(StageVote::SerializationConflict), "a phantom insert into a collection observed absent must abort the stage \ vote via the collection floor" ); @@ -825,8 +825,8 @@ fn absent_document_read_without_matching_insert_still_commits() { ); assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); assert_eq!( - staged.read_set_valid, - Some(true), + staged.stage_vote, + Some(StageVote::Commit), "an absent-document read must NOT over-abort when no insert lands in its \ collection" ); @@ -871,7 +871,7 @@ fn already_ordered_stage_and_flush_run_past_their_deadline() { )), ); assert_eq!(staged.status, Status::Ok, "late stage must run: {staged:?}"); - assert_eq!(staged.read_set_valid, Some(true)); + assert_eq!(staged.stage_vote, Some(StageVote::Commit)); let flush_plan = resolved_flush(&mut core, &mut tx, &mut rx, 9, 0); let flush = send_request( diff --git a/nodedb/tests/inproc/cases/request_tracker_backpressure.rs b/nodedb/tests/inproc/cases/request_tracker_backpressure.rs index 706e8dca0..011e3a8e5 100644 --- a/nodedb/tests/inproc/cases/request_tracker_backpressure.rs +++ b/nodedb/tests/inproc/cases/request_tracker_backpressure.rs @@ -26,7 +26,7 @@ fn partial(id: u64, data: &[u8]) -> Response { watermark_lsn: Lsn::ZERO, read_version_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, write_set: Vec::new(), } } diff --git a/nodedb/tests/inproc/cases/startup_gate_pgwire.rs b/nodedb/tests/inproc/cases/startup_gate_pgwire.rs index b4425008c..71a56fdd1 100644 --- a/nodedb/tests/inproc/cases/startup_gate_pgwire.rs +++ b/nodedb/tests/inproc/cases/startup_gate_pgwire.rs @@ -86,7 +86,7 @@ fn spawn_fake_data_plane(mut data_side: CoreChannelDataSide) { watermark_lsn: Lsn::ZERO, read_version_lsn: Lsn::ZERO, error_code: None, - read_set_valid: None, + stage_vote: None, write_set: Vec::new(), }, }; diff --git a/nodedb/tests/inproc/cases/write_admission_fence.rs b/nodedb/tests/inproc/cases/write_admission_fence.rs index 5a4687eca..2a34d4148 100644 --- a/nodedb/tests/inproc/cases/write_admission_fence.rs +++ b/nodedb/tests/inproc/cases/write_admission_fence.rs @@ -18,12 +18,12 @@ //! test suite); these tests assert the gate's fence DECISION, which is the piece //! this change introduces. -use std::collections::BTreeSet; +use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; use nodedb::bridge::dispatch::Dispatcher; use nodedb::control::cluster::calvin::scheduler::lock_manager::{ - AcquireOutcome, LockKey, LockManager, TxnId, + AcquireOutcome, LockKey, LockManager, LockMode, TxnId, }; use nodedb::control::server::shared::write_admission::{ WriteAdmission, WriteTarget, admit, cp_routed_to_calvin, @@ -120,7 +120,7 @@ async fn fence_write_blocks_behind_held_commit_lock() { // A pending commit (a normal Calvin-band txn) holds the fence on key K. let commit_txn = TxnId::new(5, 0); - let held: BTreeSet = [kv_lock_key(coll, b"K")].into(); + let held: BTreeMap = [(kv_lock_key(coll, b"K"), LockMode::Exclusive)].into(); assert_eq!( lm.lock().expect("lm").acquire(commit_txn, held), AcquireOutcome::Ready, @@ -290,7 +290,7 @@ async fn fast_path_drop_delivers_promoted_scheduler_txn() { // K and queues behind the fast-path holder (Blocked) — the exact contention // the old "fast-path keys are never contended" assumption ignored. let scheduler_txn = TxnId::new(7, 0); - let want: BTreeSet = [kv_lock_key(coll, b"K")].into(); + let want: BTreeMap = [(kv_lock_key(coll, b"K"), LockMode::Exclusive)].into(); assert_eq!( lm.lock().expect("lm").acquire(scheduler_txn, want.clone()), AcquireOutcome::Blocked, From ec72aad088e33a4d0aab5b287d199e536116e64d Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Fri, 9 Oct 2026 16:23:54 +0800 Subject: [PATCH 03/19] fix(calvin): gate data-group proposals on the leader and bound resends A replicated write now passes the write gate on the data-group leader before its entry enters the log, whether the proposal is local or forwarded. The gate takes the write's lock keys on the vShard's Calvin lock table and holds them until the leader starts the entry's apply, so a Calvin transaction that needs those keys orders after the write. - A contended write that a Calvin transaction can sequence is refused with RouteToSequencer, and the proposer submits it to the sequencer. Appends never route. - Any other contended write waits, holding no key, until the keys are free or the proposer's deadline passes (AdmissionTimedOut). A forwarded entry carries what remains of that deadline. - The gate itself moves into the host crate behind the DataProposeGate trait, split into admission keys, holds, and wait modules. The transport no longer resends a request that was written and got no answer unless RaftRpc::resend_safe holds. Any other such request ends as the new ClusterError::Unanswered, an unknown outcome, and the gateway reports a lost answer instead of running the request twice. Proposals carry a deadline, and Calvin submits draw on a shared budget. --- .../tests/common_suite/cases/mod.rs | 1 + .../write_admission_leader_gate_cluster.rs | 129 +++++ nodedb-cluster/src/circuit_breaker.rs | 2 + .../src/distributed_array/scatter.rs | 2 +- nodedb-cluster/src/error.rs | 27 +- nodedb-cluster/src/lib.rs | 8 +- nodedb-cluster/src/raft_loop/builder.rs | 14 + .../src/raft_loop/handle_rpc/dispatch.rs | 2 +- .../src/raft_loop/handle_rpc/plan_dispatch.rs | 52 +- nodedb-cluster/src/raft_loop/loop_core.rs | 8 + nodedb-cluster/src/raft_loop/mod.rs | 2 + nodedb-cluster/src/raft_loop/proposals.rs | 67 ++- nodedb-cluster/src/raft_loop/propose_gate.rs | 89 +++ nodedb-cluster/src/rpc_codec/data_propose.rs | 122 +++- nodedb-cluster/src/rpc_codec/mod.rs | 2 + nodedb-cluster/src/rpc_codec/resend.rs | 116 ++++ .../src/rpc_codec/shard_error/convert.rs | 2 + .../src/rpc_codec/shard_error/wire.rs | 4 + .../src/transport/client/attempt.rs | 123 +++- nodedb/src/bootstrap/constraint_reconcile.rs | 3 +- nodedb/src/control/backup/cut.rs | 3 +- .../control/backup/restore/array_reissue.rs | 8 +- .../control/backup/restore/crdt_reissue.rs | 4 +- nodedb/src/control/backup/restore/durable.rs | 4 +- .../cluster/array_cluster_exec/dispatch.rs | 7 +- .../control/cluster/array_cluster_helpers.rs | 11 +- .../control/cluster/array_executor/write.rs | 11 +- .../calvin/scheduler/driver/helpers.rs | 12 + .../calvin/scheduler/lock/manager/release.rs | 19 + .../calvin/scheduler/lock/manager/types.rs | 14 + nodedb/src/control/cluster/read_index.rs | 1 + .../src/control/cluster/start_raft/hooks.rs | 9 + .../control/cluster/start_raft/loop_build.rs | 1 + nodedb/src/control/cluster/start_raft/mod.rs | 3 + .../cluster/start_raft/propose_error.rs | 111 +++- .../cluster/start_raft/proposer_wiring.rs | 21 +- .../cluster/start_raft/sequencer_route.rs | 99 ++++ .../src/control/cluster/start_raft_helpers.rs | 5 + nodedb/src/control/crdt_admission.rs | 7 +- .../apply_loop/pipeline.rs | 19 +- nodedb/src/control/exec_receiver/executor.rs | 2 + nodedb/src/control/gateway/dispatch_local.rs | 10 +- nodedb/src/control/gateway/dispatch_remote.rs | 86 ++- .../maintenance/clone_materializer/chained.rs | 3 +- .../src/control/metadata_proposer/handle.rs | 1 + nodedb/src/control/orchestrated_write.rs | 3 +- .../src/control/planner/calvin/reservation.rs | 23 +- .../control/planner/calvin/submit/assign.rs | 19 +- .../control/planner/calvin/submit/budget.rs | 71 +++ .../src/control/planner/calvin/submit/mod.rs | 1 + .../control/planner/calvin/submit/routed.rs | 19 +- .../planner/calvin/submit/unique_claims.rs | 35 +- .../server/dispatch_utils/durable_write.rs | 4 +- .../submit_write/funnel/admission.rs | 58 +- .../submit_write/funnel/driver.rs | 77 ++- .../pgwire/handler/dispatch/replicated.rs | 10 +- .../shared/session/commit/single_shard.rs | 3 +- .../shared/write_admission/admission_keys.rs | 524 ++++++++++++++++++ .../server/shared/write_admission/gate.rs | 445 ++++++++++----- .../server/shared/write_admission/holds.rs | 209 +++++++ .../shared/write_admission/leader_gate.rs | 294 ++++++++++ .../shared/write_admission/lock_keys.rs | 15 +- .../server/shared/write_admission/mod.rs | 16 +- .../server/shared/write_admission/route.rs | 47 +- .../server/shared/write_admission/wait.rs | 152 +++++ .../control/server/shuffle/aggregator_hook.rs | 12 +- .../control/server/shuffle/consumer_hook.rs | 13 +- .../server/surrogate_exchange/authority.rs | 3 +- nodedb/src/control/state/calvin_local.rs | 5 + .../control/wal_replication/decode/entry.rs | 9 + .../src/control/wal_replication/decode/mod.rs | 2 +- nodedb/src/control/wal_replication/mod.rs | 1 + nodedb/src/control/wal_replication/propose.rs | 20 +- nodedb/src/control/write_resolve/propose.rs | 3 +- nodedb/src/event/topic/publish.rs | 22 +- nodedb/tests/inproc/cases/mod.rs | 1 + .../inproc/cases/write_admission_fence.rs | 114 +++- .../cases/write_admission_leader_gate.rs | 231 ++++++++ 78 files changed, 3307 insertions(+), 400 deletions(-) create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/write_admission_leader_gate_cluster.rs create mode 100644 nodedb-cluster/src/raft_loop/propose_gate.rs create mode 100644 nodedb-cluster/src/rpc_codec/resend.rs create mode 100644 nodedb/src/control/cluster/start_raft/sequencer_route.rs create mode 100644 nodedb/src/control/planner/calvin/submit/budget.rs create mode 100644 nodedb/src/control/server/shared/write_admission/admission_keys.rs create mode 100644 nodedb/src/control/server/shared/write_admission/holds.rs create mode 100644 nodedb/src/control/server/shared/write_admission/leader_gate.rs create mode 100644 nodedb/src/control/server/shared/write_admission/wait.rs create mode 100644 nodedb/tests/inproc/cases/write_admission_leader_gate.rs diff --git a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs index d40f209e2..f6240a566 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs @@ -163,3 +163,4 @@ mod vshard_names; mod webhook_sink_partial_replication; mod webhook_sink_single_owner; mod write_admission_concurrent_same_key_replay; +mod write_admission_leader_gate_cluster; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/write_admission_leader_gate_cluster.rs b/nodedb-cluster-tests/tests/common_suite/cases/write_admission_leader_gate_cluster.rs new file mode 100644 index 000000000..e399a1f94 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/write_admission_leader_gate_cluster.rs @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! An autocommit write and a Calvin cross-shard transaction on one row of a +//! replicated vShard leave the same final row on every replica. +//! +//! The data-group leader admits every replicated write through its write +//! gate before the propose. The gate takes the write's row key and its +//! collection `Intent` on the vShard's Calvin lock table, so a Calvin +//! transaction on the same row orders before or after the write on the +//! leader, never between its propose and its apply start. Followers apply +//! the committed entry in log order. +//! +//! Each round races a cross-shard transaction that writes the row against an +//! autocommit write of the row from another node. After the last round every +//! replica that hosts the row's group must hold the same image of it. + +use nodedb::types::{TenantDataSnapshot, TenantId}; + +use super::calvin_multishard_fixture::{Fixture, schemaless_ddl}; +use super::vshard_names::distinct_vshard_collections; +use crate::common::cluster_harness::TestClusterNode; +use crate::common::cluster_harness::shared_steps::key_collection; +use crate::common::pgwire_harness::raw_pgwire::RawPgConn; + +const TENANT: u64 = 1; +const ROUNDS: usize = 8; + +/// Whether the server answered `messages` with an error. +fn failed(messages: &[(u8, Vec)]) -> bool { + messages.iter().any(|(tag, _)| *tag == b'E') +} + +/// Run one cross-shard transaction that writes the row `r` of `rows` and the +/// row `s` of `other`. A refused statement or commit rolls the transaction +/// back: the race may abort it, and convergence is the property under test. +async fn calvin_round(conn: &mut RawPgConn, rows: &str, other: &str, round: usize) { + let statements = [ + "BEGIN".to_owned(), + format!("UPDATE {rows} SET v = 'calvin_{round}' WHERE id = 'r'"), + format!("UPDATE {other} SET v = 'calvin_{round}' WHERE id = 's'"), + "COMMIT".to_owned(), + ]; + for sql in &statements { + if failed(&conn.simple_query(sql).await) { + conn.simple_query("ROLLBACK").await; + return; + } + } +} + +/// Every stored row image of `collection` on `node`, from its own tenant +/// snapshot. +async fn local_row_images(node: &TestClusterNode, collection: &str) -> Vec { + let bytes = node.create_tenant_snapshot(TenantId::new(TENANT)).await; + let snapshot: TenantDataSnapshot = zerompk::from_msgpack(&bytes).unwrap_or_default(); + snapshot + .documents + .into_iter() + .filter(|(key, _)| key_collection(key) == Some(collection)) + .map(|(key, value)| { + nodedb_types::json_from_msgpack(&value) + .unwrap_or_else(|e| panic!("stored row {key} decodes: {e}")) + }) + .collect() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn autocommit_and_calvin_writes_of_one_row_converge_on_every_replica() { + let (rows, other) = distinct_vshard_collections("wag_rows", "wag_other"); + let fx = Fixture::spawn(&[schemaless_ddl(&rows), schemaless_ddl(&other)]).await; + fx.wait_group_mounted(&rows).await; + fx.wait_group_mounted(&other).await; + + let mut seed = fx.raw().await; + for sql in [ + format!("INSERT INTO {rows} {{ id: 'r', v: 'seed' }}"), + format!("INSERT INTO {other} {{ id: 's', v: 'seed' }}"), + ] { + assert!(!failed(&seed.simple_query(&sql).await), "seed: {sql}"); + } + fx.converge().await; + + // The autocommit writer runs on a node other than the transaction's + // coordinator, so its write reaches the row's leader on its own route. + let writer = (fx.coordinator + 1) % fx.cluster.nodes.len(); + let mut txn_conn = fx.raw().await; + for round in 0..ROUNDS { + let autocommit = async { + // A refused autocommit leaves the row as the transaction wrote + // it; convergence still holds. + let _ = fx.cluster.nodes[writer] + .client + .simple_query(&format!( + "UPDATE {rows} SET v = 'auto_{round}' WHERE id = 'r'" + )) + .await; + }; + tokio::join!( + calvin_round(&mut txn_conn, &rows, &other, round), + autocommit + ); + } + fx.converge().await; + + let group_id = fx.cluster.nodes[0] + .group_id_for_collection(&rows) + .expect("the row's collection maps to a data group"); + let mut images = Vec::new(); + for (idx, node) in fx.cluster.nodes.iter().enumerate() { + if !node.hosts_data_group(group_id) { + continue; + } + let node_images = local_row_images(node, &rows).await; + assert_eq!( + node_images.len(), + 1, + "node {idx} holds exactly the one row: {node_images:?}" + ); + images.push((idx, node_images)); + } + assert!(images.len() > 1, "the row's group has several replicas"); + let (first_idx, first) = &images[0]; + for (idx, image) in &images[1..] { + assert_eq!( + image, first, + "replica {idx} diverged from replica {first_idx}" + ); + } +} diff --git a/nodedb-cluster/src/circuit_breaker.rs b/nodedb-cluster/src/circuit_breaker.rs index c74ccfe39..a47380f2a 100644 --- a/nodedb-cluster/src/circuit_breaker.rs +++ b/nodedb-cluster/src/circuit_breaker.rs @@ -349,6 +349,8 @@ impl RetryPolicy { // A timed-out send may still have reached the peer, so resending // it can apply the request twice. ClusterError::ShardTimeout { .. } => false, + // The request was written, so the peer can have run it. + ClusterError::Unanswered { .. } => false, ClusterError::Raft(_) | ClusterError::VShardNotMapped { .. } | ClusterError::GroupNotFound { .. } diff --git a/nodedb-cluster/src/distributed_array/scatter.rs b/nodedb-cluster/src/distributed_array/scatter.rs index a697059f0..4c58f172d 100644 --- a/nodedb-cluster/src/distributed_array/scatter.rs +++ b/nodedb-cluster/src/distributed_array/scatter.rs @@ -94,7 +94,7 @@ fn counts_against_breaker(err: &ClusterError) -> bool { | ClusterError::StreamTerminal { .. } => false, // An unresponsive peer is exactly what the breaker exists to shed // load from, so a shard timeout counts like any other liveness failure. - ClusterError::ShardTimeout { .. } => true, + ClusterError::ShardTimeout { .. } | ClusterError::Unanswered { .. } => true, ClusterError::Raft(_) | ClusterError::VShardNotMapped { .. } | ClusterError::GroupNotFound { .. } diff --git a/nodedb-cluster/src/error.rs b/nodedb-cluster/src/error.rs index 5ee76b7b1..26e716a99 100644 --- a/nodedb-cluster/src/error.rs +++ b/nodedb-cluster/src/error.rs @@ -25,6 +25,20 @@ pub enum CalvinError { /// for the full variant set. #[error("sequencer error: {0}")] Sequencer(#[from] crate::calvin::sequencer::error::SequencerError), + + /// The data-group leader's write gate found a key of the proposed write + /// held, and a Calvin transaction can sequence the write. The proposer + /// submits the write to the Calvin sequencer instead. + #[error( + "a lock key of the proposed write is held on the data-group leader; \ + submit the write through the Calvin sequencer" + )] + RouteToSequencer, + + /// The data-group leader's write gate waited for the proposed write's + /// lock keys until its admission deadline. + #[error("the lock keys of the proposed write stayed held past the leader's admission deadline")] + AdmissionTimedOut, } /// Error emitted when applying or validating a `MigrationCheckpoint` entry. @@ -103,6 +117,14 @@ pub enum ClusterError { #[error("shard {vshard_id} RPC timed out after {elapsed_ms}ms")] ShardTimeout { vshard_id: u32, elapsed_ms: u64 }, + /// A request reached the wire to `node_id`, and no answer came back. + /// + /// The stream or connection failed, or the read timed out, after the + /// request was written. The peer can have run it, so its outcome is + /// unknown. A resend can run it twice. + #[error("request to node {node_id} was sent and got no answer: {detail}")] + Unanswered { node_id: u64, detail: String }, + /// Terminal error carried by a streaming `ExecuteStreamEnd` frame. /// /// Preserves the typed shape end-to-end so the coordinator can map a @@ -258,7 +280,10 @@ impl ClusterError { pub fn is_link_failure(&self) -> bool { matches!( self, - Self::Transport { .. } | Self::CircuitOpen { .. } | Self::NodeUnreachable { .. } + Self::Transport { .. } + | Self::Unanswered { .. } + | Self::CircuitOpen { .. } + | Self::NodeUnreachable { .. } ) } } diff --git a/nodedb-cluster/src/lib.rs b/nodedb-cluster/src/lib.rs index c6e62ea1b..5ed99f51c 100644 --- a/nodedb-cluster/src/lib.rs +++ b/nodedb-cluster/src/lib.rs @@ -118,10 +118,10 @@ pub use migration_executor::{ pub use multi_raft::{GroupStatus, MultiRaft}; pub use raft_loop::{ ApplyPermit, AssignRemoteSurrogate, AuthLeaseService, BuiltGroupSnapshot, CalvinSubmit, - CalvinSubmitInbox, CommitApplier, GroupApplyGates, InstallPermit, MetadataSnapshotCapture, - RaftLoop, ReleaseReservation, ReserveRead, ShuffleAggregator, ShuffleConsumer, ShuffleProducer, - ShuffleReceiver, SnapshotApplier, SnapshotBuilder, SnapshotQuarantineHook, - VShardEnvelopeHandler, + CalvinSubmitInbox, CommitApplier, DataProposeGate, GroupApplyGates, InstallPermit, + MetadataSnapshotCapture, ProposeHold, RaftLoop, ReleaseReservation, ReserveRead, + ShuffleAggregator, ShuffleConsumer, ShuffleProducer, ShuffleReceiver, SnapshotApplier, + SnapshotBuilder, SnapshotQuarantineHook, VShardEnvelopeHandler, }; pub use reachability::{ NoopProber, ReachabilityDriver, ReachabilityDriverConfig, ReachabilityProber, TransportProber, diff --git a/nodedb-cluster/src/raft_loop/builder.rs b/nodedb-cluster/src/raft_loop/builder.rs index c04245500..f0b31ab4b 100644 --- a/nodedb-cluster/src/raft_loop/builder.rs +++ b/nodedb-cluster/src/raft_loop/builder.rs @@ -55,6 +55,7 @@ impl RaftLoop { calvin_submit_inbox: self.calvin_submit_inbox, reserve_read: self.reserve_read, release_reservation: self.release_reservation, + data_propose_gate: self.data_propose_gate, auth_lease: self.auth_lease, snapshot_builder: self.snapshot_builder, snapshot_applier: self.snapshot_applier, @@ -228,6 +229,19 @@ impl RaftLoop { self } + /// Attach the write gate for data-group proposals (builder chain). + /// + /// The supplied implementation (backed by `nodedb`'s write-admission + /// gate) admits every proposal to a data group this node leads, local + /// or forwarded, before the entry enters the log. + pub fn with_data_propose_gate( + mut self, + gate: Arc, + ) -> Self { + self.data_propose_gate = Some(gate); + self + } + /// Attach the per-group snapshot builder for the SEND path (builder chain). /// /// The supplied implementation (backed by `nodedb`'s Data Plane snapshot diff --git a/nodedb-cluster/src/raft_loop/handle_rpc/dispatch.rs b/nodedb-cluster/src/raft_loop/handle_rpc/dispatch.rs index 8439b9128..0e072827a 100644 --- a/nodedb-cluster/src/raft_loop/handle_rpc/dispatch.rs +++ b/nodedb-cluster/src/raft_loop/handle_rpc/dispatch.rs @@ -40,7 +40,7 @@ impl RaftRpcHandler for RaftLoop { // Metadata-group proposal forwarding. RaftRpc::MetadataProposeRequest(req) => self.handle_metadata_propose_rpc(req), // Data-group proposal forwarding. - RaftRpc::DataProposeRequest(req) => self.handle_data_propose_rpc(req), + RaftRpc::DataProposeRequest(req) => self.handle_data_propose_rpc(req).await, // Read index for a node that does not lead the group. RaftRpc::ReadIndexRequest(req) => self.handle_read_index_rpc(req).await, // The leader this node knows for a group, from its own Raft state. diff --git a/nodedb-cluster/src/raft_loop/handle_rpc/plan_dispatch.rs b/nodedb-cluster/src/raft_loop/handle_rpc/plan_dispatch.rs index af6cb2e2a..d684ae418 100644 --- a/nodedb-cluster/src/raft_loop/handle_rpc/plan_dispatch.rs +++ b/nodedb-cluster/src/raft_loop/handle_rpc/plan_dispatch.rs @@ -4,12 +4,12 @@ //! VShardEnvelope routing RPC bodies. use crate::calvin::SEQUENCER_GROUP_ID; -use crate::error::{ClusterError, Result}; +use crate::error::{CalvinError, ClusterError, Result}; use crate::forward::{ChunkSink, PlanExecutor}; use crate::multi_raft::MultiRaft; use crate::rpc_codec::{ DataProposeRequest, DataProposeResponse, ExecuteRequest, MetadataProposeRequest, ProposeTarget, - RaftRpc, TypedClusterError, VShardRefusal, + RaftRpc, TypedClusterError, VShardRefusal, forwarded_deadline, }; use super::super::loop_core::{CommitApplier, RaftLoop}; @@ -48,11 +48,29 @@ impl RaftLoop { // Data-group and sequencer-group proposal forwarding — apply locally if // we lead the target group, otherwise return NotLeader with a hint so the - // forwarder can chase the redirect. - pub(super) fn handle_data_propose_rpc(&self, req: DataProposeRequest) -> Result { - let resp = { - let mut mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); - propose_forwarded(&mut mr, req) + // forwarder can chase the redirect. A data-group entry passes this + // leader's write gate first, exactly as a local proposal does. + pub(super) async fn handle_data_propose_rpc(&self, req: DataProposeRequest) -> Result { + let resp = match req.target { + ProposeTarget::VShard { + vshard_id, + deadline_remaining_ms, + } => match forwarded_deadline(deadline_remaining_ms) { + // The proposer stopped waiting: the entry is refused unproposed. + None => DataProposeResponse::refused(&ClusterError::Calvin( + CalvinError::AdmissionTimedOut, + )), + Some(deadline) => { + match self.propose_admitted(vshard_id, &req.bytes, deadline).await { + Ok((group_id, log_index)) => DataProposeResponse::ok(group_id, log_index), + Err(error) => DataProposeResponse::refused(&error), + } + } + }, + ProposeTarget::Sequencer => { + let mut mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + propose_to_sequencer(&mut mr, req.bytes) + } }; Ok(RaftRpc::DataProposeResponse(resp)) } @@ -92,19 +110,13 @@ fn vshard_answer(result: Result>) -> RaftRpc { } } -/// Propose a forwarded entry to its target group on this node. +/// Propose a forwarded sequencer entry to the sequencer group on this node. /// /// Answers `not leader` with the known leader as a hint when this node does -/// not lead the target group. -fn propose_forwarded(mr: &mut MultiRaft, req: DataProposeRequest) -> DataProposeResponse { - let proposed = match req.target { - ProposeTarget::VShard(vshard_id) => mr.propose(vshard_id, req.bytes), - ProposeTarget::Sequencer => mr - .propose_to_group(SEQUENCER_GROUP_ID, req.bytes) - .map(|log_index| (SEQUENCER_GROUP_ID, log_index)), - }; - match proposed { - Ok((group_id, log_index)) => DataProposeResponse::ok(group_id, log_index), +/// not lead the sequencer group. +fn propose_to_sequencer(mr: &mut MultiRaft, bytes: Vec) -> DataProposeResponse { + match mr.propose_to_group(SEQUENCER_GROUP_ID, bytes) { + Ok(log_index) => DataProposeResponse::ok(SEQUENCER_GROUP_ID, log_index), Err(error) => DataProposeResponse::refused(&error), } } @@ -178,7 +190,7 @@ mod tests { elect_sequencer_leader(&mut mr); let before = mr.last_log_index(SEQUENCER_GROUP_ID).unwrap_or(0); - let resp = propose_forwarded(&mut mr, sequencer_request()); + let resp = propose_to_sequencer(&mut mr, sequencer_request().bytes); assert!(resp.success, "{}", resp.error_message); assert_eq!(resp.group_id, SEQUENCER_GROUP_ID); @@ -191,7 +203,7 @@ mod tests { let dir = tempfile::tempdir().expect("tempdir"); let mut mr = multi_raft_with_sequencer(dir.path()); - let resp = propose_forwarded(&mut mr, sequencer_request()); + let resp = propose_to_sequencer(&mut mr, sequencer_request().bytes); assert!(!resp.success); assert_eq!( diff --git a/nodedb-cluster/src/raft_loop/loop_core.rs b/nodedb-cluster/src/raft_loop/loop_core.rs index 65061f2c1..20d6c1ca9 100644 --- a/nodedb-cluster/src/raft_loop/loop_core.rs +++ b/nodedb-cluster/src/raft_loop/loop_core.rs @@ -232,6 +232,13 @@ pub struct RaftLoop { /// configured" error. pub(super) release_reservation: Option>, + /// Optional write gate for data-group proposals on their leader. When + /// set (by the `nodedb` binary via `with_data_propose_gate`), a proposal + /// to a group this node leads takes its lock keys through it before the + /// entry enters the log. Cluster-only tests leave this `None`, which + /// proposes every entry ungated. + pub(super) data_propose_gate: Option>, + /// Optional authorization lease service. When set (by the `nodedb` /// binary via `with_auth_lease`), lease renewals and authorization /// barriers are answered through it. @@ -344,6 +351,7 @@ impl RaftLoop { calvin_submit_inbox: None, reserve_read: None, release_reservation: None, + data_propose_gate: None, auth_lease: None, snapshot_builder: None, snapshot_applier: None, diff --git a/nodedb-cluster/src/raft_loop/mod.rs b/nodedb-cluster/src/raft_loop/mod.rs index ebfc4b702..039f16c38 100644 --- a/nodedb-cluster/src/raft_loop/mod.rs +++ b/nodedb-cluster/src/raft_loop/mod.rs @@ -32,6 +32,7 @@ pub mod loop_core; mod membership_convergence; mod placement_reconcile; pub mod proposals; +pub mod propose_gate; mod read_index; mod routing_persist; mod run; @@ -48,3 +49,4 @@ pub use hooks::{ }; pub use in_flight_snapshots::{InFlightSnapshotGuard, InFlightSnapshots}; pub use loop_core::{CommitApplier, RaftLoop, VShardEnvelopeHandler}; +pub use propose_gate::{DataProposeGate, ProposeHold}; diff --git a/nodedb-cluster/src/raft_loop/proposals.rs b/nodedb-cluster/src/raft_loop/proposals.rs index 88ef3d8e7..8bb2a28da 100644 --- a/nodedb-cluster/src/raft_loop/proposals.rs +++ b/nodedb-cluster/src/raft_loop/proposals.rs @@ -9,6 +9,11 @@ use crate::error::Result; use super::loop_core::{CommitApplier, RaftLoop}; use crate::forward::PlanExecutor; +/// How long a forwarded data propose waits for the leader's reply past the +/// proposer's deadline. The leader's gate stops at that deadline, so the +/// margin covers the round trip and the leader's verdict arrives. +const FORWARD_REPLY_MARGIN: std::time::Duration = std::time::Duration::from_secs(2); + impl RaftLoop { /// Propose a command to the Raft group owning the given vShard. /// @@ -223,15 +228,30 @@ impl RaftLoop { /// a `DataProposeRequest` over QUIC. The receiving leader applies the /// proposal locally and returns `(group_id, log_index)`. /// + /// The leader admits the entry through its write gate before the entry + /// enters its log (see [`super::propose_gate`]). A gate refusal returns + /// as `ClusterError::Calvin`. The gate waits no later than `deadline`, + /// the caller's deadline. A forward carries what remains of it. + /// + /// The call ends no later than `deadline` plus [`FORWARD_REPLY_MARGIN`]. + /// `CalvinError::AdmissionTimedOut` means the entry was not proposed. + /// `ClusterError::ShardTimeout` means the leader's reply never came, so + /// the leader can have proposed the entry. `ClusterError::Unanswered` + /// means the request was written and its stream failed, with the same + /// unknown outcome. The transport never resends a written forward. A + /// `ClusterError::Transport` from the send means the request never went + /// out whole, so nothing was proposed. + /// /// On `NotLeader { leader_hint: None, .. }` (election in progress) the /// call returns the original `NotLeader` error so the caller can retry. pub async fn propose_via_data_leader( &self, vshard_id: u32, - data: Vec, + data: &[u8], + deadline: tokio::time::Instant, ) -> Result<(u64, u64)> { // First, try a local propose. - match self.propose(vshard_id, data.clone()) { + match self.propose_admitted(vshard_id, data, deadline).await { Ok(pair) => Ok(pair), Err(crate::error::ClusterError::Raft(nodedb_raft::RaftError::NotLeader { leader_hint, @@ -254,19 +274,30 @@ impl RaftLoop { )); } // Otherwise forward to the hinted leader. - self.forward_data_propose(leader_id, vshard_id, data).await + self.forward_data_propose(leader_id, vshard_id, data.to_vec(), deadline) + .await } Err(other) => Err(other), } } - /// Send a `DataProposeRequest` to `leader_id`. + /// Send a `DataProposeRequest` carrying the budget left to `deadline` to + /// `leader_id`, and wait for its verdict until `deadline` plus + /// [`FORWARD_REPLY_MARGIN`]. async fn forward_data_propose( &self, leader_id: u64, vshard_id: u32, data: Vec, + deadline: tokio::time::Instant, ) -> Result<(u64, u64)> { + let deadline_remaining_ms = crate::rpc_codec::remaining_budget_ms(deadline); + // The deadline passed before the forward left: nothing is proposed. + if deadline_remaining_ms == 0 { + return Err(crate::error::ClusterError::Calvin( + crate::error::CalvinError::AdmissionTimedOut, + )); + } { let topo = self.topology.read().unwrap_or_else(|p| p.into_inner()); let Some(node) = topo.get_node(leader_id) else { @@ -289,10 +320,32 @@ impl RaftLoop { let req = crate::rpc_codec::RaftRpc::DataProposeRequest(crate::rpc_codec::DataProposeRequest { - target: crate::rpc_codec::ProposeTarget::VShard(vshard_id), + target: crate::rpc_codec::ProposeTarget::VShard { + vshard_id, + deadline_remaining_ms, + }, bytes: data, }); - let resp = self.transport.send_rpc(leader_id, req).await?; + // The generic short RPC timeout ends the call before a contended + // write's gate wait does. The reply wait follows the forwarded budget. + let reply_budget = + std::time::Duration::from_millis(deadline_remaining_ms) + FORWARD_REPLY_MARGIN; + let sent = tokio::time::timeout( + reply_budget, + self.transport + .send_rpc_with_read_timeout(leader_id, req, reply_budget), + ) + .await; + let resp = match sent { + Ok(answer) => answer?, + // The leader can have proposed the entry: its outcome is unknown. + Err(_) => { + return Err(crate::error::ClusterError::ShardTimeout { + vshard_id, + elapsed_ms: u64::try_from(reply_budget.as_millis()).unwrap_or(u64::MAX), + }); + } + }; match resp { crate::rpc_codec::RaftRpc::DataProposeResponse(r) => { if r.success { @@ -312,7 +365,7 @@ impl RaftLoop { Err(r.refusal_error()) } } - other => Err(crate::error::ClusterError::Transport { + other => Err(crate::error::ClusterError::Codec { detail: format!("data propose forward: unexpected response variant {other:?}"), }), } diff --git a/nodedb-cluster/src/raft_loop/propose_gate.rs b/nodedb-cluster/src/raft_loop/propose_gate.rs new file mode 100644 index 000000000..198a854d5 --- /dev/null +++ b/nodedb-cluster/src/raft_loop/propose_gate.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The data-group leader's write gate. +//! +//! The leader of a vShard's data group admits every proposal to that group +//! through the host's write gate before the entry enters its log. The gate +//! takes the write's lock keys on the vShard's Calvin lock table. A Calvin +//! transaction that needs one of those keys then waits until this leader +//! started the entry's apply, so its own effects order after the write. +//! +//! A local proposal and a forwarded one pass the same gate: both reach the +//! leader through [`RaftLoop::propose_admitted`]. `nodedb-cluster` cannot +//! depend on `nodedb`, so the gate lives in the host crate behind +//! [`DataProposeGate`]. + +use crate::error::Result; +use crate::forward::PlanExecutor; + +use super::loop_core::{CommitApplier, RaftLoop}; + +/// The lock keys one admitted proposal holds. Dropping it releases them. +pub trait ProposeHold: Send { + /// The proposal landed at `log_index` of `group_id`. The host keeps the + /// keys until this node starts the apply of that entry. + fn landed(self: Box, group_id: u64, log_index: u64); +} + +/// The host's write gate for data-group proposals on their leader. +#[async_trait::async_trait] +pub trait DataProposeGate: Send + Sync + 'static { + /// Admit the encoded entry `entry`, proposed to the group of + /// `vshard_id`, on this leader. A contended entry waits for its keys + /// until `deadline`, the proposer's own deadline. + /// + /// - `Ok(None)`: the entry takes no lock key. + /// - `Ok(Some(hold))`: the entry holds its keys until `hold` releases + /// them. + /// - `Err(ClusterError::Calvin(CalvinError::RouteToSequencer))`: a held + /// key blocks the write, and the proposer submits it to the Calvin + /// sequencer. + /// - `Err(ClusterError::Calvin(CalvinError::AdmissionTimedOut))`: the + /// keys stayed held until `deadline`. Nothing is proposed. + async fn admit( + &self, + vshard_id: u32, + entry: &[u8], + deadline: tokio::time::Instant, + ) -> Result>>; +} + +impl RaftLoop { + /// Propose `data` to the group of `vshard_id` on this node. When this + /// node leads the group, the write gate admits the entry first, waiting + /// no later than `deadline`, the proposer's deadline. + /// + /// A node that does not lead the group proposes with no gate. The + /// propose refuses with `NotLeader`, and the caller forwards the entry to + /// the leader, which gates it there. A propose that fails after the gate + /// admitted the entry drops its hold, which releases the keys. + pub(super) async fn propose_admitted( + &self, + vshard_id: u32, + data: &[u8], + deadline: tokio::time::Instant, + ) -> Result<(u64, u64)> { + let hold = match &self.data_propose_gate { + Some(gate) if self.leads_group_of(vshard_id) => { + gate.admit(vshard_id, data, deadline).await? + } + Some(_) | None => None, + }; + let (group_id, log_index) = self.propose(vshard_id, data.to_vec())?; + if let Some(hold) = hold { + hold.landed(group_id, log_index); + } + Ok((group_id, log_index)) + } + + /// Whether this node leads the data group that owns `vshard_id`. + fn leads_group_of(&self, vshard_id: u32) -> bool { + let mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + let group_id = mr + .routing() + .read() + .unwrap_or_else(|p| p.into_inner()) + .group_for_vshard(vshard_id); + group_id.is_ok_and(|group_id| mr.is_group_leader(group_id)) + } +} diff --git a/nodedb-cluster/src/rpc_codec/data_propose.rs b/nodedb-cluster/src/rpc_codec/data_propose.rs index dd2244da1..63ba4c3cc 100644 --- a/nodedb-cluster/src/rpc_codec/data_propose.rs +++ b/nodedb-cluster/src/rpc_codec/data_propose.rs @@ -15,14 +15,42 @@ use crate::error::{ClusterError, Result}; /// The Raft group a forwarded proposal is for. #[derive(Debug, Clone, Copy, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] pub enum ProposeTarget { - /// The data group that owns this vShard. The bytes are a serialized + /// The data group that owns `vshard_id`. The bytes are a serialized /// `ReplicatedEntry`. - VShard(u32), + /// + /// `deadline_remaining_ms` is what remains of the proposer's deadline + /// when it sends the request. The leader's write gate stops waiting for + /// the entry's keys once this budget runs out. `0` means the deadline + /// passed, and the leader refuses the entry unproposed. + VShard { + vshard_id: u32, + deadline_remaining_ms: u64, + }, /// The Calvin sequencer group. The bytes are a msgpack-encoded /// `SequencerEntry`. Sequencer, } +/// The budget a forward carries for `deadline`: the milliseconds left now. +/// `0` when the deadline passed. +pub fn remaining_budget_ms(deadline: tokio::time::Instant) -> u64 { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + u64::try_from(remaining.as_millis()).unwrap_or(u64::MAX) +} + +/// The leader's deadline for a forward that carries `deadline_remaining_ms`. +/// `None` when the budget is spent. +/// +/// The budget is used as it arrived. The proposer resolved it once, from the +/// statement's own timeout or its node default. Re-deciding it against the +/// leader's default gives one statement two budgets. +pub fn forwarded_deadline(deadline_remaining_ms: u64) -> Option { + match deadline_remaining_ms { + 0 => None, + ms => Some(tokio::time::Instant::now() + std::time::Duration::from_millis(ms)), + } +} + /// Forward an opaque proposal payload to the leader of its target group. #[derive(Debug, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] pub struct DataProposeRequest { @@ -39,6 +67,12 @@ pub enum ForwardedProposeRefusal { NotLeader, /// A leadership transfer of the target group is in flight. LeadershipTransferInProgress, + /// The leader's write gate routes the write through the Calvin + /// sequencer. + RouteToSequencer, + /// The proposer's deadline passed before the leader's write gate + /// admitted the write. The leader did not propose it. + AdmissionTimedOut, /// Any other failure; `error_message` describes it. Failed, } @@ -82,6 +116,12 @@ impl DataProposeResponse { None, 0, ), + ClusterError::Calvin(crate::error::CalvinError::RouteToSequencer) => { + (ForwardedProposeRefusal::RouteToSequencer, None, 0) + } + ClusterError::Calvin(crate::error::CalvinError::AdmissionTimedOut) => { + (ForwardedProposeRefusal::AdmissionTimedOut, None, 0) + } // A refusal with no retry contract. The forwarding node reads it // as a transport error carrying the leader's message. ClusterError::Raft( @@ -105,6 +145,7 @@ impl DataProposeResponse { | ClusterError::GhostNotFound { .. } | ClusterError::Transport { .. } | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. } | ClusterError::StreamTerminal { .. } | ClusterError::Storage { .. } | ClusterError::DataPlane { .. } @@ -147,8 +188,9 @@ impl DataProposeResponse { } /// The typed error a refused response stands for on the forwarding node. - /// A refusal the leader typed keeps its Raft error; any other stays a - /// transport error carrying the leader's message. + /// A refusal the leader typed keeps its Raft error. Any other becomes + /// `ClusterError::RemoteUntyped` with the leader's message. The leader + /// answered, so it is never a link failure. pub fn refusal_error(&self) -> ClusterError { match self.refusal { Some(ForwardedProposeRefusal::NotLeader) => { @@ -160,8 +202,14 @@ impl DataProposeResponse { Some(ForwardedProposeRefusal::LeadershipTransferInProgress) => { ClusterError::Raft(nodedb_raft::RaftError::LeadershipTransferInProgress) } - Some(ForwardedProposeRefusal::Failed) | None => ClusterError::Transport { - detail: format!("data propose forward failed: {}", self.error_message), + Some(ForwardedProposeRefusal::RouteToSequencer) => { + ClusterError::Calvin(crate::error::CalvinError::RouteToSequencer) + } + Some(ForwardedProposeRefusal::AdmissionTimedOut) => { + ClusterError::Calvin(crate::error::CalvinError::AdmissionTimedOut) + } + Some(ForwardedProposeRefusal::Failed) | None => ClusterError::RemoteUntyped { + detail: format!("data propose forward refused: {}", self.error_message), }, } } @@ -235,10 +283,41 @@ mod tests { assert_eq!(req.bytes, vec![1, 2, 3]); } + /// The vShard target keeps the proposer's remaining budget. + #[test] + fn vshard_target_keeps_its_deadline_across_the_wire() { + let target = ProposeTarget::VShard { + vshard_id: 42, + deadline_remaining_ms: 1_500, + }; + let req = roundtrip(target); + assert_eq!(req.target, target); + } + + /// A spent budget gives the leader no deadline, so it proposes nothing. + #[test] + fn a_spent_budget_has_no_leader_deadline() { + assert_eq!(forwarded_deadline(0), None); + } + + /// The leader's deadline ends no later than the budget the forward + /// carries. #[test] - fn vshard_target_survives_the_wire() { - let req = roundtrip(ProposeTarget::VShard(42)); - assert_eq!(req.target, ProposeTarget::VShard(42)); + fn the_leader_deadline_ends_within_the_forwarded_budget() { + let budget = std::time::Duration::from_millis(50); + let latest = tokio::time::Instant::now() + budget; + let deadline = forwarded_deadline(50).expect("a live budget"); + let received = tokio::time::Instant::now() + budget; + assert!(deadline <= received); + assert!(deadline >= latest); + } + + /// A passed deadline leaves no budget to forward. + #[test] + fn a_passed_deadline_leaves_no_budget() { + assert_eq!(remaining_budget_ms(tokio::time::Instant::now()), 0); + let live = tokio::time::Instant::now() + std::time::Duration::from_secs(10); + assert!(remaining_budget_ms(live) > 9_000); } fn refusal_across_the_wire(error: ClusterError) -> ClusterError { @@ -281,9 +360,30 @@ mod tests { )); } + /// A write-gate refusal keeps its Calvin error across the wire, so the + /// forwarding node routes the write or reports the deadline. + #[test] + fn a_write_gate_refusal_keeps_its_calvin_error_across_the_wire() { + let routed = refusal_across_the_wire(ClusterError::Calvin( + crate::error::CalvinError::RouteToSequencer, + )); + assert!(matches!( + routed, + ClusterError::Calvin(crate::error::CalvinError::RouteToSequencer) + )); + let timed_out = refusal_across_the_wire(ClusterError::Calvin( + crate::error::CalvinError::AdmissionTimedOut, + )); + assert!(matches!( + timed_out, + ClusterError::Calvin(crate::error::CalvinError::AdmissionTimedOut) + )); + } + #[test] - fn any_other_refusal_stays_a_transport_error() { + fn any_other_refusal_is_an_untyped_answer() { let error = refusal_across_the_wire(ClusterError::VShardNotMapped { vshard_id: 7 }); - assert!(matches!(error, ClusterError::Transport { .. })); + assert!(matches!(error, ClusterError::RemoteUntyped { .. })); + assert!(!error.is_link_failure()); } } diff --git a/nodedb-cluster/src/rpc_codec/mod.rs b/nodedb-cluster/src/rpc_codec/mod.rs index 83ade303b..aab00dcab 100644 --- a/nodedb-cluster/src/rpc_codec/mod.rs +++ b/nodedb-cluster/src/rpc_codec/mod.rs @@ -27,6 +27,7 @@ pub mod raft_msgs; pub mod raft_rpc; pub mod read_index; pub mod request_refusal; +pub mod resend; pub mod reservation; pub mod shard_error; pub mod shuffle; @@ -54,6 +55,7 @@ pub use data_plane_error::{ }; pub use data_propose::{ DataProposeRequest, DataProposeResponse, ForwardedProposeRefusal, ProposeTarget, + forwarded_deadline, remaining_budget_ms, }; pub use execute::{ DescriptorVersionEntry, ExecuteRequest, ExecuteResponse, ExecuteStreamChunk, ExecuteStreamEnd, diff --git a/nodedb-cluster/src/rpc_codec/resend.rs b/nodedb-cluster/src/rpc_codec/resend.rs new file mode 100644 index 000000000..c600e4283 --- /dev/null +++ b/nodedb-cluster/src/rpc_codec/resend.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Which requests the transport can send a second time after a written +//! attempt got no answer. + +use super::raft_rpc::RaftRpc; + +impl RaftRpc { + /// Whether a second run of this request on the peer leaves the same + /// state and answer as one run. + /// + /// The transport resends a written request with no answer only when + /// this holds. The peer can have run the first copy. Every request that + /// proposes, submits, executes, assigns or moves state answers `false`, + /// and a lost answer ends as an unknown outcome. A reply variant never + /// goes out as a request, so it answers `false`. + pub fn resend_safe(&self) -> bool { + match self { + // Raft log matching makes a repeated append a no-op. + RaftRpc::AppendEntriesRequest(_) + // A voter grants at most one candidate per term, and a repeated + // request from that candidate gets the same grant. + | RaftRpc::RequestVoteRequest(_) + // A pre-vote changes no Raft state. + | RaftRpc::PreVoteRequest(_) + // Reads of the peer's state. + | RaftRpc::Ping(_) + | RaftRpc::ReadIndexRequest(_) + | RaftRpc::LeaderStatusRequest(_) => true, + // A second copy starts a second election. + RaftRpc::TimeoutNowRequest(_) + // The receiver takes a chunk only at its next expected offset, + // so it refuses a repeat of a chunk that landed. + | RaftRpc::InstallSnapshotRequest(_) + // Membership, topology and lease changes. + | RaftRpc::JoinRequest(_) + | RaftRpc::TopologyUpdate(_) + | RaftRpc::AuthLeaseRenewRequest(_) + | RaftRpc::AuthBarrierRequest(_) + // Proposals, submits and executions. + | RaftRpc::VShardEnvelope(_) + | RaftRpc::MetadataProposeRequest(_) + | RaftRpc::ExecuteRequest(_) + | RaftRpc::ExecuteStreamRequest(_) + | RaftRpc::ShufflePushRequest(_) + | RaftRpc::ShufflePushChunk(_) + | RaftRpc::ShufflePushEnd(_) + | RaftRpc::ShuffleProduceRequest(_) + | RaftRpc::ShuffleConsumeRequest(_) + | RaftRpc::ShuffleAggregateConsumeRequest(_) + | RaftRpc::AssignSurrogateRequest(_) + | RaftRpc::SubmitCalvinTxnRequest(_) + | RaftRpc::SubmitCalvinInboxRequest(_) + | RaftRpc::CalvinPartsRequest(_) + | RaftRpc::ReserveReadRequest(_) + | RaftRpc::ReleaseReservationRequest(_) + | RaftRpc::DataProposeRequest(_) + // Replies. + | RaftRpc::AppendEntriesResponse(_) + | RaftRpc::RequestVoteResponse(_) + | RaftRpc::PreVoteResponse(_) + | RaftRpc::InstallSnapshotResponse(_) + | RaftRpc::JoinResponse(_) + | RaftRpc::Pong(_) + | RaftRpc::TopologyAck(_) + | RaftRpc::MetadataProposeResponse(_) + | RaftRpc::ExecuteResponse(_) + | RaftRpc::ExecuteStreamChunk(_) + | RaftRpc::ExecuteStreamEnd(_) + | RaftRpc::ShuffleProduceResponse(_) + | RaftRpc::ShuffleConsumeResponse(_) + | RaftRpc::ShuffleAggregateConsumeResponse(_) + | RaftRpc::AssignSurrogateResponse(_) + | RaftRpc::SubmitCalvinTxnResponse(_) + | RaftRpc::SubmitCalvinInboxResponse(_) + | RaftRpc::CalvinPartsResponse(_) + | RaftRpc::ReserveReadResponse(_) + | RaftRpc::ReleaseReservationResponse(_) + | RaftRpc::DataProposeResponse(_) + | RaftRpc::ReadIndexResponse(_) + | RaftRpc::LeaderStatusResponse(_) + | RaftRpc::AuthLeaseRenewResponse(_) + | RaftRpc::AuthBarrierResponse(_) + | RaftRpc::VShardRefusal(_) + | RaftRpc::FrameRefused(_) + | RaftRpc::RequestRefused(_) => false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::rpc_codec::{DataProposeRequest, PingRequest, ProposeTarget}; + + #[test] + fn a_data_propose_is_never_resent() { + let propose = RaftRpc::DataProposeRequest(DataProposeRequest { + target: ProposeTarget::VShard { + vshard_id: 3, + deadline_remaining_ms: 100, + }, + bytes: vec![1, 2, 3], + }); + assert!(!propose.resend_safe()); + } + + #[test] + fn a_ping_is_resent() { + let ping = RaftRpc::Ping(PingRequest { + sender_id: 1, + topology_version: 0, + }); + assert!(ping.resend_safe()); + } +} diff --git a/nodedb-cluster/src/rpc_codec/shard_error/convert.rs b/nodedb-cluster/src/rpc_codec/shard_error/convert.rs index a0b1f4c9f..71cccdbaf 100644 --- a/nodedb-cluster/src/rpc_codec/shard_error/convert.rs +++ b/nodedb-cluster/src/rpc_codec/shard_error/convert.rs @@ -42,6 +42,7 @@ impl From for ShardErrorWire { Self::GhostNotFound { node_id, shard_id } } ClusterError::Transport { detail } => Self::Transport { detail }, + ClusterError::Unanswered { node_id, detail } => Self::Unanswered { node_id, detail }, ClusterError::ShardTimeout { vshard_id, elapsed_ms, @@ -175,6 +176,7 @@ impl From for ClusterError { Self::GhostNotFound { node_id, shard_id } } ShardErrorWire::Transport { detail } => Self::Transport { detail }, + ShardErrorWire::Unanswered { node_id, detail } => Self::Unanswered { node_id, detail }, ShardErrorWire::ShardTimeout { vshard_id, elapsed_ms, diff --git a/nodedb-cluster/src/rpc_codec/shard_error/wire.rs b/nodedb-cluster/src/rpc_codec/shard_error/wire.rs index 6c585d80a..d85cfb6ce 100644 --- a/nodedb-cluster/src/rpc_codec/shard_error/wire.rs +++ b/nodedb-cluster/src/rpc_codec/shard_error/wire.rs @@ -123,4 +123,8 @@ pub enum ShardErrorWire { error: TypedClusterError, detail: String, }, + Unanswered { + node_id: u64, + detail: String, + }, } diff --git a/nodedb-cluster/src/transport/client/attempt.rs b/nodedb-cluster/src/transport/client/attempt.rs index 888e3eadb..800562a8d 100644 --- a/nodedb-cluster/src/transport/client/attempt.rs +++ b/nodedb-cluster/src/transport/client/attempt.rs @@ -9,6 +9,11 @@ //! typed refusal included. A refusal goes back to the caller as its typed //! error and is not resent. //! +//! The peer can have run a request written before a link failure. The +//! transport resends such a request only when [`RaftRpc::resend_safe`] +//! holds. Any other request ends as `ClusterError::Unanswered`, an unknown +//! outcome. +//! //! A link failure drops the pooled connection only when the connection //! itself failed. Other sends share that connection, so one stream's //! failure must not cut them off. The connection fails when: @@ -47,22 +52,50 @@ enum Attempt { /// A link failure, and the pooled connection it condemns. #[derive(Debug)] struct LinkFailure { - error: ClusterError, + error: Box, /// Stable id of the connection to drop from the pool. `None` when no /// connection was obtained or the connection still works. condemned: Option, + /// Whether the whole request was written before the failure. The peer + /// can have run a written request. + written: bool, } impl LinkFailure { - /// A failure that leaves the pool as it is. + /// A failure before the request went out that leaves the pool as it is. fn keep_connection(error: ClusterError) -> Self { Self { - error, + error: Box::new(error), condemned: None, + written: false, + } + } + + /// The error of a written request that got no answer from `target`. + fn into_unanswered(self, target: u64) -> ClusterError { + ClusterError::Unanswered { + node_id: target, + detail: self.error.to_string(), } } } +/// Whether a failed attempt is sent again. +/// +/// `Ok` holds the error kept while the next attempt goes out. `Err` ends the +/// call. A written request that is not [`RaftRpc::resend_safe`] can have run +/// on the peer. It is never resent and ends as `ClusterError::Unanswered`. +fn resend_verdict(failure: LinkFailure, target: u64, resend_safe: bool) -> Result { + if failure.written && !resend_safe { + return Err(failure.into_unanswered(target)); + } + if RetryPolicy::is_retryable(&failure.error) { + Ok(*failure.error) + } else { + Err(*failure.error) + } +} + /// Sort the outcome of one send into an [`Attempt`]. /// /// The outer error is a link failure. The inner result is what the peer @@ -110,6 +143,7 @@ impl NexarTransport { // Each retry wraps it in a fresh envelope so the seq advances // per attempt — a retry is a new frame, not a replayed frame. let inner = rpc_codec::encode(&rpc, &self.auth.epoch)?; + let resend_safe = rpc.resend_safe(); let mut last_err = None; for attempt in 0..self.retry_policy.max_attempts { @@ -124,10 +158,9 @@ impl NexarTransport { match self.attempt(target, &inner, read_timeout, admission).await { Attempt::Answered(answer) => return answer, Attempt::FrameRefused(e) => last_err = Some(e), - Attempt::LinkFailed(failure) if RetryPolicy::is_retryable(&failure.error) => { - last_err = Some(failure.error) + Attempt::LinkFailed(failure) => { + last_err = Some(resend_verdict(failure, target, resend_safe)?) } - Attempt::LinkFailed(failure) => return Err(failure.error), } } @@ -152,7 +185,7 @@ impl NexarTransport { { Attempt::Answered(answer) => answer, Attempt::FrameRefused(e) => Err(e), - Attempt::LinkFailed(failure) => Err(failure.error), + Attempt::LinkFailed(failure) => Err(*failure.error), } } @@ -206,23 +239,26 @@ impl NexarTransport { let stable_id = conn.stable_id(); if let Err(error) = self.verify_connection_target(&conn, target) { return Err(LinkFailure { - error, + error: Box::new(error), condemned: Some(stable_id), + written: false, }); } let received_before = received_datagrams(&conn); self.exchange(&conn, target, inner, read_timeout) .await .map_err(|failure| { - let (error, timed_out) = match failure { - StreamFailure::ReadTimeout(error) => (error, true), - StreamFailure::Other(error) => (error, false), + let (error, timed_out, written) = match failure { + StreamFailure::Unsent(error) => (error, false, false), + StreamFailure::ReadTimeout(error) => (error, true, true), + StreamFailure::Unanswered(error) => (error, false, true), }; let heard = received_datagrams(&conn) > received_before; let broken = condemns_connection(timed_out, conn.close_reason().is_some(), heard); LinkFailure { - error, + error: Box::new(error), condemned: broken.then_some(stable_id), + written, } }) } @@ -236,19 +272,21 @@ impl NexarTransport { read_timeout: Duration, ) -> std::result::Result, StreamFailure> { let (mut send, mut recv) = conn.open_bi().await.map_err(|e| { - StreamFailure::Other(ClusterError::Transport { + StreamFailure::Unsent(ClusterError::Transport { detail: format!("open_bi to node {target}: {e}"), }) })?; - let envelope = self.wrap_inner(inner).map_err(StreamFailure::Other)?; + let envelope = self.wrap_inner(inner).map_err(StreamFailure::Unsent)?; + // A failed `write_all` left part of the frame unbuffered, so the peer + // cannot hold the whole request. Once it returns, the peer can. send.write_all(&envelope).await.map_err(|e| { - StreamFailure::Other(ClusterError::Transport { + StreamFailure::Unsent(ClusterError::Transport { detail: format!("write to node {target}: {e}"), }) })?; send.finish().map_err(|e| { - StreamFailure::Other(ClusterError::Transport { + StreamFailure::Unanswered(ClusterError::Transport { detail: format!("finish send to node {target}: {e}"), }) })?; @@ -264,7 +302,7 @@ impl NexarTransport { ), }) })? - .map_err(StreamFailure::Other)?; + .map_err(StreamFailure::Unanswered)?; let Some(response_envelope) = response_envelope else { return Ok(Err(ClusterError::RemoteUntyped { detail: format!("node {target} finished the stream without a reply"), @@ -281,10 +319,15 @@ impl NexarTransport { /// How one stream on a pooled connection failed. #[derive(Debug)] enum StreamFailure { - /// The reply did not arrive within the read timeout. + /// The request did not go out whole: the stream did not open, the + /// envelope did not build, or the write failed. + Unsent(ClusterError), + /// The request was written, and the reply did not arrive within the + /// read timeout. ReadTimeout(ClusterError), - /// Opening, writing or reading the stream failed. - Other(ClusterError), + /// The request was written, and finishing the stream or reading the + /// reply failed. + Unanswered(ClusterError), } /// Whether a stream failure condemns its connection. @@ -355,6 +398,46 @@ mod tests { assert_eq!(client.circuit_breaker().failure_count(1), 1); } + fn lost(written: bool) -> LinkFailure { + LinkFailure { + error: Box::new(ClusterError::Transport { + detail: "connection lost".into(), + }), + condemned: None, + written, + } + } + + /// A propose that went out can have run on the peer. It is never + /// resent, and it ends as an unknown outcome. + #[test] + fn a_written_request_that_is_not_resend_safe_ends_unanswered() { + match resend_verdict(lost(true), 2, false) { + Err(ClusterError::Unanswered { node_id: 2, .. }) => {} + other => panic!("expected Unanswered, got {other:?}"), + } + } + + /// A request that never went out ran nowhere, so it is resent. + #[test] + fn an_unsent_request_is_resent() { + for resend_safe in [false, true] { + assert!(matches!( + resend_verdict(lost(false), 2, resend_safe), + Ok(ClusterError::Transport { .. }) + )); + } + } + + /// A request whose second run is harmless is resent after a write. + #[test] + fn a_written_resend_safe_request_is_resent() { + assert!(matches!( + resend_verdict(lost(true), 2, true), + Ok(ClusterError::Transport { .. }) + )); + } + #[test] fn a_typed_refusal_is_a_final_answer() { let refusal = RaftRpc::RequestRefused(RequestRefusal::from(ClusterError::GroupNotFound { diff --git a/nodedb/src/bootstrap/constraint_reconcile.rs b/nodedb/src/bootstrap/constraint_reconcile.rs index 7dff168fd..90e2b28b1 100644 --- a/nodedb/src/bootstrap/constraint_reconcile.rs +++ b/nodedb/src/bootstrap/constraint_reconcile.rs @@ -169,7 +169,8 @@ pub async fn reconcile_once( }, ); - match propose_replicated_entry(shared, &proposer, entry).await { + let deadline = crate::control::wal_replication::statement_propose_deadline(shared); + match propose_replicated_entry(shared, &proposer, entry, deadline).await { Ok(_) => { // Record only on commit. A transient / NotLeader error // leaves the map untouched so the next tick retries. diff --git a/nodedb/src/control/backup/cut.rs b/nodedb/src/control/backup/cut.rs index 02cb48bde..ed70e709a 100644 --- a/nodedb/src/control/backup/cut.rs +++ b/nodedb/src/control/backup/cut.rs @@ -144,6 +144,7 @@ async fn cut_data_groups( capture: Option<&CutCaptureRequest>, ) -> Result<(), Error> { let proposer = state.async_raft_proposer()?; + let deadline = crate::control::wal_replication::statement_propose_deadline(state); let barriers = futures::future::join_all(barrier_vshards(state).into_iter().map( |(group_id, vshard_id)| { // The barrier orders every entry of its group, whatever database @@ -160,7 +161,7 @@ async fn cut_data_groups( }, ); async move { - propose_replicated_entry(state, proposer, entry) + propose_replicated_entry(state, proposer, entry, deadline) .await .map_err(|error| (group_id, error)) } diff --git a/nodedb/src/control/backup/restore/array_reissue.rs b/nodedb/src/control/backup/restore/array_reissue.rs index 4700a445c..87b176994 100644 --- a/nodedb/src/control/backup/restore/array_reissue.rs +++ b/nodedb/src/control/backup/restore/array_reissue.rs @@ -322,7 +322,13 @@ impl CellWrite<'_> { })? .with_event_source(EventSource::Restore) .with_restore_id(self.target.restore_id); - crate::control::wal_replication::propose_replicated_entry(state, proposer, entry).await?; + crate::control::wal_replication::propose_replicated_entry( + state, + proposer, + entry, + crate::control::wal_replication::statement_propose_deadline(state), + ) + .await?; Ok(()) } } diff --git a/nodedb/src/control/backup/restore/crdt_reissue.rs b/nodedb/src/control/backup/restore/crdt_reissue.rs index 70cff33a7..9016067ab 100644 --- a/nodedb/src/control/backup/restore/crdt_reissue.rs +++ b/nodedb/src/control/backup/restore/crdt_reissue.rs @@ -51,7 +51,9 @@ async fn reissue_crdt_collection( })? .with_event_source(EventSource::Restore) .with_restore_id(target.restore_id); - crate::control::wal_replication::propose_replicated_entry(state, proposer, entry).await?; + let deadline = crate::control::wal_replication::statement_propose_deadline(state); + crate::control::wal_replication::propose_replicated_entry(state, proposer, entry, deadline) + .await?; Ok(()) } diff --git a/nodedb/src/control/backup/restore/durable.rs b/nodedb/src/control/backup/restore/durable.rs index ea4d7fa95..7fb39e1cb 100644 --- a/nodedb/src/control/backup/restore/durable.rs +++ b/nodedb/src/control/backup/restore/durable.rs @@ -67,8 +67,10 @@ pub(crate) async fn reissue_plan_durably( // fire again for it. .with_event_source(crate::event::EventSource::Restore) .with_restore_id(target.restore_id); + let deadline = crate::control::wal_replication::statement_propose_deadline(state); let (_, write_version) = - crate::control::wal_replication::propose_replicated_entry(state, proposer, entry).await?; + crate::control::wal_replication::propose_replicated_entry(state, proposer, entry, deadline) + .await?; tracing::debug!( collection, vshard_id = vshard.as_u32(), diff --git a/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs b/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs index 2efa24a78..36f7ebc6f 100644 --- a/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs +++ b/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs @@ -135,8 +135,11 @@ impl NexarArrayDispatch { self.transport.send_rpc(node_id, rpc), ) .await - .map_err(|_| nodedb_cluster::error::ClusterError::Transport { - detail: format!("array shard RPC timeout ({timeout_ms}ms) to node {node_id}"), + // The request can have reached the shard, so a timeout is never + // resent: `call` retries only a `Transport` error. + .map_err(|_| nodedb_cluster::error::ClusterError::ShardTimeout { + vshard_id: req.vshard_id, + elapsed_ms: timeout_ms, })??; match resp_rpc { diff --git a/nodedb/src/control/cluster/array_cluster_helpers.rs b/nodedb/src/control/cluster/array_cluster_helpers.rs index bbd41789b..9300a5694 100644 --- a/nodedb/src/control/cluster/array_cluster_helpers.rs +++ b/nodedb/src/control/cluster/array_cluster_helpers.rs @@ -99,10 +99,13 @@ pub(super) fn cluster_err(e: ClusterError) -> Error { match e { // A shard did not answer within its timeout: surface as a deterministic // deadline rather than an opaque internal error, matching the - // `TypedClusterError::DeadlineExceeded` mapping used elsewhere. - ClusterError::ShardTimeout { .. } => Error::DeadlineExceeded { - request_id: crate::types::RequestId::new(0), - }, + // `TypedClusterError::DeadlineExceeded` mapping used elsewhere. A + // sent request with no answer can have run, so it takes the same class. + ClusterError::ShardTimeout { .. } | ClusterError::Unanswered { .. } => { + Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + } + } // A shard's Data-Plane verdict keeps its code, so the statement // renders the SQLSTATE a single-node execution renders. ClusterError::DataPlane { code } => Error::DataPlane(code.into()), diff --git a/nodedb/src/control/cluster/array_executor/write.rs b/nodedb/src/control/cluster/array_executor/write.rs index 1bd825663..261b68c82 100644 --- a/nodedb/src/control/cluster/array_executor/write.rs +++ b/nodedb/src/control/cluster/array_executor/write.rs @@ -125,9 +125,14 @@ impl DataPlaneArrayExecutor { })?; let (apply_payload, _write_version) = - crate::control::wal_replication::propose_replicated_entry(&self.state, proposer, entry) - .await - .map_err(|e| execution_error(&format!("{op_label} raft propose"), e))?; + crate::control::wal_replication::propose_replicated_entry( + &self.state, + proposer, + entry, + crate::control::wal_replication::statement_propose_deadline(&self.state), + ) + .await + .map_err(|e| execution_error(&format!("{op_label} raft propose"), e))?; let affected = require_affected_count(&apply_payload).map_err(|e| ClusterError::Storage { detail: format!("{op_label}: {e}"), diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs index 4e0d1e198..7eaaf45e7 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/helpers.rs @@ -79,6 +79,18 @@ pub(crate) fn expand_rw_set(txn: &SequencedTxn) -> BTreeMap { keys } +/// Expand write key sets alone into the moded lock request a transaction of +/// those writes takes, by the rules of [`expand_rw_set`]. The +/// write-admission gate locks an autocommit write with it, so the write and +/// a Calvin transaction of the same plan take the same keys. +pub(crate) fn expand_write_key_sets(sets: &[EngineKeySet]) -> BTreeMap { + let mut keys = BTreeMap::new(); + for ks in sets { + add_key_set(&mut keys, ks, LockMode::Exclusive); + } + keys +} + /// Add the keys of `ks` to `keys`. `row_mode` is `Shared` for a read set and /// `Exclusive` for a write set. fn add_key_set(keys: &mut BTreeMap, ks: &EngineKeySet, row_mode: LockMode) { diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs index 302d9129a..f7f1ca556 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/release.rs @@ -33,6 +33,9 @@ impl LockManager { entry.waiters.retain(|(w, _)| *w != txn); self.promote_waiters(key, &mut newly_promoted); } + if !keys.is_empty() { + self.released.notify_waiters(); + } newly_promoted.into_iter().collect() } @@ -156,6 +159,22 @@ mod tests { ); } + /// A release wakes a waiter that enabled the release signal before it. + #[tokio::test] + async fn release_wakes_the_release_signal() { + let mut lm = LockManager::new(); + let holder = txn(1, 0); + assert_eq!(lm.acquire(holder, keyset(&["k"])), AcquireOutcome::Ready); + let signal = lm.release_signal(); + let notified = signal.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + lm.release(holder); + tokio::time::timeout(std::time::Duration::from_secs(5), notified) + .await + .expect("a release wakes the release signal"); + } + #[test] fn release_preserves_fifo_waiter_order() { let mut lm = LockManager::new(); diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs index 2de091c68..0ec4039d6 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/types.rs @@ -4,6 +4,9 @@ //! acquire/release paths share. use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use tokio::sync::Notify; use crate::control::cluster::calvin::scheduler::lock::lock_entry::{LockEntry, LockMode}; use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId}; @@ -36,6 +39,9 @@ pub struct LockManager { /// key is granted on the promotion path inside `release`. pub(in crate::control::cluster::calvin::scheduler::lock) pending_keys: BTreeMap>, + /// Woken on every release. A write-admission waiter that holds no key + /// probes the table again after each wake. + pub(in crate::control::cluster::calvin::scheduler::lock) released: Arc, } /// How a requester resolves a request that conflicts on at least one key. @@ -56,8 +62,16 @@ impl LockManager { table: BTreeMap::new(), held_locks: BTreeMap::new(), pending_keys: BTreeMap::new(), + released: Arc::new(Notify::new()), } } + + /// The signal every release wakes. A waiter enables its `notified()` + /// future before it probes the table, so no release between the probe + /// and the wait goes unseen. + pub fn release_signal(&self) -> Arc { + Arc::clone(&self.released) + } } impl Default for LockManager { diff --git a/nodedb/src/control/cluster/read_index.rs b/nodedb/src/control/cluster/read_index.rs index d2dcd06ec..86493a307 100644 --- a/nodedb/src/control/cluster/read_index.rs +++ b/nodedb/src/control/cluster/read_index.rs @@ -118,6 +118,7 @@ fn refusal_of(error: ClusterError) -> ReadIndexRefusal { | ClusterError::GhostNotFound { .. } | ClusterError::Transport { .. } | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. } | ClusterError::StreamTerminal { .. } | ClusterError::Storage { .. } | ClusterError::DataPlane { .. } diff --git a/nodedb/src/control/cluster/start_raft/hooks.rs b/nodedb/src/control/cluster/start_raft/hooks.rs index 1033f62a2..bf5875f2e 100644 --- a/nodedb/src/control/cluster/start_raft/hooks.rs +++ b/nodedb/src/control/cluster/start_raft/hooks.rs @@ -28,6 +28,8 @@ pub(super) struct Hooks { pub(super) calvin_submit_inbox: Arc, pub(super) reserve_read: Arc, pub(super) release_reservation: Arc, + /// The write gate a data-group leader runs on every proposal to its group. + pub(super) data_propose_gate: Arc, /// The sequencer group's kept snapshot, which its own compaction writes. pub(super) sequencer_snapshots: Arc, @@ -182,6 +184,12 @@ pub(super) async fn build_hooks( crate::control::server::reservation::RegistryReleaseReservation::new(shared.clone()), ); + // Leader write gate: when this node leads a data group, every proposal + // to it, local or forwarded, takes its lock keys on the vShard's Calvin + // lock table before the entry enters the log. + let data_propose_gate: Arc = + Arc::new(crate::control::server::shared::write_admission::LeaderWriteGate::new(shared)); + Ok(Hooks { quarantine_hook, snapshot_builder, @@ -195,6 +203,7 @@ pub(super) async fn build_hooks( calvin_submit_inbox, reserve_read, release_reservation, + data_propose_gate, sequencer_snapshots, }) } diff --git a/nodedb/src/control/cluster/start_raft/loop_build.rs b/nodedb/src/control/cluster/start_raft/loop_build.rs index 498d0004f..e51fc48b1 100644 --- a/nodedb/src/control/cluster/start_raft/loop_build.rs +++ b/nodedb/src/control/cluster/start_raft/loop_build.rs @@ -156,6 +156,7 @@ pub(super) async fn build_raft_loop( .with_calvin_submit_inbox(hooks.calvin_submit_inbox) .with_reserve_read(hooks.reserve_read) .with_release_reservation(hooks.release_reservation) + .with_data_propose_gate(hooks.data_propose_gate) .with_auth_lease(lease_service) .with_data_dir(data_dir.to_path_buf()) .with_snapshot_chunk_bytes(snapshot_chunk_bytes) diff --git a/nodedb/src/control/cluster/start_raft/mod.rs b/nodedb/src/control/cluster/start_raft/mod.rs index 76838459d..04e22ebf9 100644 --- a/nodedb/src/control/cluster/start_raft/mod.rs +++ b/nodedb/src/control/cluster/start_raft/mod.rs @@ -14,6 +14,8 @@ //! cluster subsystems. //! - [`proposer_wiring`]: install the sync/async Raft proposer + compactor //! closures and spawn the apply loop. +//! - [`sequencer_route`]: run a proposal the leader's write gate refused +//! through the Calvin sequencer. //! - [`observability`]: publish observability handles and spawn the tick //! loop, sequencer service, RPC server, and health monitor. @@ -24,5 +26,6 @@ mod loop_build; mod observability; mod propose_error; mod proposer_wiring; +mod sequencer_route; pub use core::start_raft; diff --git a/nodedb/src/control/cluster/start_raft/propose_error.rs b/nodedb/src/control/cluster/start_raft/propose_error.rs index 35240fa44..7f73da055 100644 --- a/nodedb/src/control/cluster/start_raft/propose_error.rs +++ b/nodedb/src/control/cluster/start_raft/propose_error.rs @@ -2,20 +2,31 @@ //! The error an async Raft propose returns to its statement. -use nodedb_cluster::ClusterError; +use nodedb_cluster::{CalvinError, ClusterError}; use nodedb_raft::RaftError; +use crate::bridge::envelope::ErrorCode; use crate::types::VShardId; +/// The error of a write whose deadline passed before any leader proposed it. +/// +/// A definite refusal: nothing entered the log, so nothing applies. A client +/// sees the same deadline SQLSTATE as an unknown outcome. A caller that +/// matches the code can tell the two apart. +pub(super) fn expired_before_propose() -> crate::Error { + crate::Error::DataPlane(ErrorCode::ExpiredBeforeExecution) +} + /// The error an async propose returns for a cluster error. /// /// A group with no leader to take the proposal right now accepts the same /// proposal once it has one, so the proposal is retried: /// [`crate::Error::NoLeader`]. That covers an election, a leadership transfer /// in flight, a leader that stepped down after this node or a forwarding node -/// chose it, and a vShard whose owner is moving. A forwarded refusal arrives -/// here with its typed Raft error (`DataProposeResponse::refusal_error`). A -/// typed verdict keeps its class. Every other failure is final here. +/// chose it, a vShard whose owner is moving, and a leader this node cannot +/// reach. A forwarded refusal arrives here with its typed Raft error +/// (`DataProposeResponse::refusal_error`). A typed verdict keeps its class. +/// Every other failure is final here. pub(super) fn async_propose_error(vshard_id: u32, error: ClusterError) -> crate::Error { match error { ClusterError::Raft( @@ -23,15 +34,27 @@ pub(super) fn async_propose_error(vshard_id: u32, error: ClusterError) -> crate: ) | ClusterError::ReadIndexNotLeader { .. } | ClusterError::MigrationInProgress { .. } - | ClusterError::WrongOwner { .. } => crate::Error::NoLeader { + | ClusterError::WrongOwner { .. } + // The forward never reached the leader whole: the leader is not in + // this node's topology, or the connect, stream open or write failed. + // Nothing was proposed. A written forward ends as `Unanswered`. + | ClusterError::Transport { .. } => crate::Error::NoLeader { vshard_id: VShardId::new(vshard_id), }, - // The forward did not answer before its timeout. The statement's - // deadline class, the same class the array fan-out gives it. - ClusterError::ShardTimeout { .. } => crate::Error::DeadlineExceeded { - request_id: crate::types::RequestId::new(0), - }, + // The forward did not answer before its timeout. The leader can have + // proposed the write, so its outcome is unknown: the statement's + // deadline class, the same class the array fan-out gives it. A + // forward whose stream failed after the request was written can + // also have been proposed, so it takes the same class. + ClusterError::ShardTimeout { .. } | ClusterError::Unanswered { .. } => { + crate::Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + } + } ClusterError::DataPlane { code } => crate::Error::DataPlane(code.into()), + // The leader's write gate waited for the write's lock keys until the + // caller's deadline, and proposed nothing. + ClusterError::Calvin(CalvinError::AdmissionTimedOut) => expired_before_propose(), ClusterError::ShardExecution { error, .. } | ClusterError::StreamTerminal { error, .. } => { crate::Error::from(*error) } @@ -53,7 +76,6 @@ pub(super) fn async_propose_error(vshard_id: u32, error: ClusterError) -> crate: | ClusterError::MigrationPauseBudgetExceeded { .. } | ClusterError::NodeUnreachable { .. } | ClusterError::GhostNotFound { .. } - | ClusterError::Transport { .. } | ClusterError::Storage { .. } | ClusterError::Codec { .. } | ClusterError::UnsupportedWireVersion { .. } @@ -113,15 +135,66 @@ mod tests { )); } + /// An admission timeout proposed nothing: a definite refusal, never an + /// unknown outcome. #[test] - fn a_forward_timeout_is_a_deadline() { - let error = ClusterError::ShardTimeout { - vshard_id: 3, - elapsed_ms: 50, + fn an_admission_timeout_is_a_definite_refusal() { + let error = async_propose_error(3, ClusterError::Calvin(CalvinError::AdmissionTimedOut)); + let crate::Error::DataPlane(code) = &error else { + panic!("expected a Data-Plane verdict, got {error:?}"); }; - assert!(matches!( - async_propose_error(3, error), - crate::Error::DeadlineExceeded { .. } - )); + assert_eq!(code, &ErrorCode::ExpiredBeforeExecution); + assert!(crate::control::server::dispatch_utils::write_definitely_not_applied(code)); + } + + /// A forward with no reply can have been proposed. Its timeout stays an + /// unknown outcome, never a definite refusal. + #[test] + fn a_forward_timeout_is_an_unknown_outcome() { + let error = async_propose_error( + 3, + ClusterError::ShardTimeout { + vshard_id: 3, + elapsed_ms: 50, + }, + ); + assert!(matches!(error, crate::Error::DeadlineExceeded { .. })); + assert!( + !crate::control::server::dispatch_utils::write_definitely_not_applied( + &ErrorCode::from(error) + ) + ); + } + + /// A forward whose stream failed after the request was written can have + /// been proposed. It is an unknown outcome, never an internal error. + #[test] + fn a_sent_forward_with_no_answer_is_an_unknown_outcome() { + let error = async_propose_error( + 3, + ClusterError::Unanswered { + node_id: 2, + detail: "connection lost".into(), + }, + ); + assert!(matches!(error, crate::Error::DeadlineExceeded { .. })); + assert!( + !crate::control::server::dispatch_utils::write_definitely_not_applied( + &ErrorCode::from(error) + ) + ); + } + + /// A forward that never went out proposed nothing. The proposal is + /// retried, never reported as an unknown outcome. + #[test] + fn an_unsent_forward_is_retried() { + let error = async_propose_error( + 3, + ClusterError::Transport { + detail: "connect to node 2 refused".into(), + }, + ); + assert!(matches!(error, crate::Error::NoLeader { .. })); } } diff --git a/nodedb/src/control/cluster/start_raft/proposer_wiring.rs b/nodedb/src/control/cluster/start_raft/proposer_wiring.rs index 9f66a278c..26b71c031 100644 --- a/nodedb/src/control/cluster/start_raft/proposer_wiring.rs +++ b/nodedb/src/control/cluster/start_raft/proposer_wiring.rs @@ -238,14 +238,23 @@ fn install_async_proposer( detail: "raft propose (async): cluster not running".into(), })?; // The attempt gets only what remains of the caller's deadline. + // A deadline that passed before the propose refuses the write. if tokio::time::Instant::now() >= deadline { - return Err(propose_deadline_exceeded()); + return Err(super::propose_error::expired_before_propose()); } - let (group_id, log_index) = - tokio::time::timeout_at(deadline, rl.propose_via_data_leader(vshard_id, data)) - .await - .map_err(|_| propose_deadline_exceeded())? - .map_err(|e| async_propose_error(vshard_id, e))?; + // The leader's write gate stops at `deadline`. The call waits + // a bounded margin past it for the leader's verdict. An + // unproposed write then reports a refusal, not an unknown outcome. + let proposed = rl.propose_via_data_leader(vshard_id, &data, deadline).await; + let (group_id, log_index) = match proposed { + Ok(at) => at, + // The leader's write gate found a key of the write held. + // The Calvin sequencer orders it behind the holder. + Err(nodedb_cluster::ClusterError::Calvin( + nodedb_cluster::CalvinError::RouteToSequencer, + )) => return Ok(super::sequencer_route::routed_write(&state_weak, data)), + Err(e) => return Err(async_propose_error(vshard_id, e)), + }; // Register the waiter with the proposer's idempotency // key. The apply path compares against the committed diff --git a/nodedb/src/control/cluster/start_raft/sequencer_route.rs b/nodedb/src/control/cluster/start_raft/sequencer_route.rs new file mode 100644 index 000000000..1209a2336 --- /dev/null +++ b/nodedb/src/control/cluster/start_raft/sequencer_route.rs @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A proposal the data-group leader's write gate routed to the Calvin +//! sequencer. +//! +//! The leader refuses the propose with `RouteToSequencer` when a key of the +//! write is held and a Calvin transaction can sequence it. The proposer then +//! submits the same write, decoded from the entry it encoded, through the +//! scheduler. The scheduler queues it FIFO behind the holder on every +//! replica. Its answer stands for the apply the propose would have returned. + +use std::sync::Weak; + +use crate::bridge::envelope::{Response, Status}; +use crate::control::server::shared::write_admission::{bare_ok_response, route_write_to_calvin}; +use crate::control::state::SharedState; +use crate::control::wal_replication::ReplicatedEntry; +use crate::control::wal_replication::decode::decode_parsed_entry; +use crate::control::wal_replication::{AppliedWait, ProposedWrite}; +use crate::types::RequestId; + +/// The proposal of the encoded entry `data`, run through the Calvin +/// sequencer. No entry landed in the data-group log. +pub(super) fn routed_write(state: &Weak, data: Vec) -> ProposedWrite { + let state = state.clone(); + let applied: AppliedWait = Box::pin(async move { + let state = state.upgrade().ok_or_else(|| crate::Error::Internal { + detail: "calvin route of a gated proposal: node is shutting down".into(), + })?; + let entry = ReplicatedEntry::from_bytes(&data).ok_or_else(|| crate::Error::Internal { + detail: "calvin route of a gated proposal: the proposer's own entry does not decode" + .into(), + })?; + let Some((database_id, (tenant_id, vshard_id, plan, _))) = decode_parsed_entry(&entry)? + else { + return Err(crate::Error::Internal { + detail: "calvin route of a gated proposal: the entry carries no plan, yet the \ + leader's write gate named it sequenceable" + .into(), + }); + }; + let event_source = crate::event::EventSource::from(entry.event_source); + let response = route_write_to_calvin( + &state, + tenant_id, + database_id, + vshard_id, + plan, + event_source, + ) + .await? + .unwrap_or_else(|| bare_ok_response(RequestId::new(0))); + applied_result(&response) + }); + ProposedWrite { at: None, applied } +} + +/// The apply result a routed write's answer stands for: its payload and +/// written version, or its typed Data-Plane verdict. +fn applied_result(response: &Response) -> crate::Result<(Vec, crate::types::Lsn)> { + if response.status == Status::Ok { + return Ok((response.payload.to_vec(), response.read_version_lsn)); + } + match response.error_code.as_deref() { + Some(code) => Err(crate::Error::DataPlane(code.clone())), + None => Err(crate::Error::Internal { + detail: "calvin route of a gated proposal: the scheduler answered an error with no \ + error code" + .into(), + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bridge::envelope::ErrorCode; + + #[test] + fn an_ok_answer_carries_its_payload_and_version() { + let mut response = bare_ok_response(RequestId::new(1)); + response.payload = crate::bridge::envelope::Payload::from_vec(vec![7]); + response.read_version_lsn = crate::types::Lsn::new(9); + let (payload, version) = applied_result(&response).expect("ok"); + assert_eq!(payload, vec![7]); + assert_eq!(version, crate::types::Lsn::new(9)); + } + + #[test] + fn an_error_answer_keeps_its_verdict() { + let mut response = bare_ok_response(RequestId::new(1)); + response.status = Status::Error; + response.error_code = Some(Box::new(ErrorCode::OllpRetryRequired)); + assert!(matches!( + applied_result(&response), + Err(crate::Error::DataPlane(ErrorCode::OllpRetryRequired)) + )); + } +} diff --git a/nodedb/src/control/cluster/start_raft_helpers.rs b/nodedb/src/control/cluster/start_raft_helpers.rs index bfcece5ec..067464de0 100644 --- a/nodedb/src/control/cluster/start_raft_helpers.rs +++ b/nodedb/src/control/cluster/start_raft_helpers.rs @@ -102,6 +102,11 @@ fn retire_vshard_schedulers( .lock() .unwrap_or_else(|p| p.into_inner()) .remove(&vshard); + // The write gate's holds on the stopped lock table order nothing. + shared + .calvin + .admission_holds + .release_vshard(crate::types::VShardId::new(vshard)); calvin_completion_registry.unregister_verdict_signal_sender(vshard); // A cut on this node waits only on the schedulers it runs. shared.calvin.cuts.unregister(vshard); diff --git a/nodedb/src/control/crdt_admission.rs b/nodedb/src/control/crdt_admission.rs index 05ba8e0cb..517d1c9df 100644 --- a/nodedb/src/control/crdt_admission.rs +++ b/nodedb/src/control/crdt_admission.rs @@ -582,7 +582,12 @@ async fn apply_fenced( .with_event_source(workflow.event_source); let outcome = tokio::time::timeout( workflow.timeout, - crate::control::wal_replication::propose_replicated_entry(workflow.state, raw, entry), + crate::control::wal_replication::propose_replicated_entry( + workflow.state, + raw, + entry, + tokio::time::Instant::now() + workflow.timeout, + ), ) .await .map_err(|_| crate::Error::CrdtAdmissionTimeout { diff --git a/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs b/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs index 89dccd45d..e2aa68472 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs @@ -124,6 +124,9 @@ impl<'a> Pipeline<'a> { (finished.group_id, finished.log_index, handled) } }; + // The entry's enqueue returned, or its exclusive apply finished: + // every entry of the group through it reached its core. + release_admission_holds(self.ctx.state, group_id, log_index); if !handled { // Every enqueue and apply the pipeline runs has a slot in its // group's lane in the matching state: the pump pushes both @@ -261,6 +264,7 @@ impl<'a> Pipeline<'a> { ); let (state, blocks, user_write) = match prepared { Prepared::Concluded(outcome) => { + release_admission_holds(shared, group_id, log_index); let user_write = matches!(outcome, EntryOutcome::Repeat) && repeat_writes; ( SlotState::Concluded(self.gate.conclude(proposal_key, false, outcome)), @@ -268,7 +272,10 @@ impl<'a> Pipeline<'a> { user_write, ) } - Prepared::Barrier => (SlotState::Barrier, false, false), + Prepared::Barrier => { + release_admission_holds(shared, group_id, log_index); + (SlotState::Barrier, false, false) + } Prepared::Enqueue(enqueue) => { self.gate.open(proposal_key); self.ctx.tracker.note_dispatched(group_id, log_index); @@ -319,6 +326,7 @@ impl<'a> Pipeline<'a> { outcome: EntryOutcome, ) -> bool { self.ctx.tracker.note_started(group_id, log_index); + release_admission_holds(self.ctx.state, group_id, log_index); self.ctx .tracker .complete_covered(group_id, log_index, proposal_key); @@ -404,6 +412,15 @@ impl<'a> Pipeline<'a> { } } +/// Release the lock keys this node's write gate holds for entries of +/// `group_id` through `log_index`: their applies started, in log order. +fn release_admission_holds(state: &SharedState, group_id: u64, log_index: u64) { + state + .calvin + .admission_holds + .release_through(group_id, log_index); +} + fn finished_event(apply: ApplyFuture<'_>) -> LoopFuture<'_> { Box::pin(apply.map(LoopEvent::Finished)) } diff --git a/nodedb/src/control/exec_receiver/executor.rs b/nodedb/src/control/exec_receiver/executor.rs index df46cd15c..7ede7385e 100644 --- a/nodedb/src/control/exec_receiver/executor.rs +++ b/nodedb/src/control/exec_receiver/executor.rs @@ -249,6 +249,8 @@ impl LocalPlanExecutor { &self.state, proposer, entry, + // The coordinator's remaining budget bounds this hop. + tokio::time::Instant::now() + deadline, ) .await { diff --git a/nodedb/src/control/gateway/dispatch_local.rs b/nodedb/src/control/gateway/dispatch_local.rs index 53c4b43e3..4c517e668 100644 --- a/nodedb/src/control/gateway/dispatch_local.rs +++ b/nodedb/src/control/gateway/dispatch_local.rs @@ -94,9 +94,13 @@ pub(super) async fn dispatch_local( )? { let proposer = shared.async_raft_proposer()?; - let (payload, write_version) = - crate::control::wal_replication::propose_replicated_entry(shared, proposer, entry) - .await?; + let (payload, write_version) = crate::control::wal_replication::propose_replicated_entry( + shared, + proposer, + entry, + crate::control::wal_replication::statement_propose_deadline(shared), + ) + .await?; return Ok(DispatchOutcome { payloads: vec![payload], // A write carries no read watermark (Lsn::ZERO); its post-write diff --git a/nodedb/src/control/gateway/dispatch_remote.rs b/nodedb/src/control/gateway/dispatch_remote.rs index 50fd74339..88186ff53 100644 --- a/nodedb/src/control/gateway/dispatch_remote.rs +++ b/nodedb/src/control/gateway/dispatch_remote.rs @@ -16,6 +16,7 @@ use nodedb_cluster::rpc_codec::{ExecuteRequest, RaftRpc}; use tracing::debug; use crate::Error; +use crate::control::security::identity::{Permission, required_permission}; use crate::control::server::result_stream::{ResultStream, RowBatch}; use crate::control::state::SharedState; use crate::types::{DatabaseId, Lsn, TenantId, TraceId, TxnId, VShardId}; @@ -133,6 +134,10 @@ pub(super) async fn dispatch_remote( } }; + // A read can be sent again after a lost answer. Any other plan can have + // run on the target, so a lost answer is an unknown outcome. + let resend_safe = required_permission(&plan) == Permission::Read; + // Encode the plan. let plan_bytes = plan_wire::encode(&plan).map_err(|e| Error::Internal { detail: format!("gateway: plan encode failed: {e}"), @@ -169,20 +174,10 @@ pub(super) async fn dispatch_remote( "gateway: dispatching ExecuteRequest to remote node" ); - let resp_rpc = transport.send_rpc(node_id, req).await.map_err(|e| { - // Transport failure means the target node is unreachable — - // we do NOT know who the new leader is. Use leader_node = 0 - // so the retry loop does NOT re-entrench the unreachable node - // as leader in the routing table. The next retry will route - // locally (leader == 0 → local) and let the local Raft state - // resolve to the actual leader. - Error::NotLeader { - vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), - leader_node: 0, - leader_addr: format!("node-{node_id} (transport error: {e})"), - leader_term: 0, - } - })?; + let resp_rpc = transport + .send_rpc(node_id, req) + .await + .map_err(|e| send_error(e, resend_safe, node_id, vshard_id))?; match resp_rpc { RaftRpc::ExecuteResponse(resp) => { @@ -394,6 +389,7 @@ fn map_stream_cluster_error(err: ClusterError, vshard_id: u64) -> Error { | ClusterError::GhostNotFound { .. } | ClusterError::Transport { .. } | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. } | ClusterError::Storage { .. } | ClusterError::Codec { .. } | ClusterError::UnsupportedWireVersion { .. } @@ -451,11 +447,73 @@ fn scoped_vshard( Ok(Some(VShardId::new(raw))) } +/// The error of a failed `ExecuteRequest` send. +/// +/// A request the target never received, and a lost answer to a read, retry as +/// `NotLeader` with `leader_node = 0`. The unreachable node is then not kept as +/// leader, and the next try routes locally to the real leader. A lost answer +/// to any other plan is an unknown outcome: the plan can have run, and a +/// second send would run it again. +fn send_error(e: ClusterError, resend_safe: bool, node_id: u64, vshard_id: u64) -> Error { + let outcome_unknown = matches!( + e, + ClusterError::Unanswered { .. } | ClusterError::ShardTimeout { .. } + ); + if outcome_unknown && !resend_safe { + return Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + }; + } + Error::NotLeader { + vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), + leader_node: 0, + leader_addr: format!("node-{node_id} (transport error: {e})"), + leader_term: 0, + } +} + #[cfg(test)] mod tests { use super::*; use nodedb_physical::physical_plan::{MetaOp, PhysicalPlan}; + fn unanswered() -> ClusterError { + ClusterError::Unanswered { + node_id: 2, + detail: "stream reset".into(), + } + } + + /// A write whose answer was lost can have applied: it is never sent again. + #[test] + fn a_lost_answer_to_a_write_is_an_unknown_outcome() { + assert!(matches!( + send_error(unanswered(), false, 2, 7), + Error::DeadlineExceeded { .. } + )); + } + + /// A read whose answer was lost retries through the leader lookup. + #[test] + fn a_lost_answer_to_a_read_retries() { + assert!(matches!( + send_error(unanswered(), true, 2, 7), + Error::NotLeader { leader_node: 0, .. } + )); + } + + /// A request that never reached the target retries, read or write. + #[test] + fn an_unsent_request_retries() { + let unsent = ClusterError::Transport { + detail: "connect refused".into(), + }; + assert!(matches!( + send_error(unsent, false, 2, 7), + Error::NotLeader { leader_node: 0, .. } + )); + } + #[test] fn a_transaction_meta_op_carries_its_route_vshard() { let plan = PhysicalPlan::Meta(MetaOp::DropTxnOverlay { diff --git a/nodedb/src/control/maintenance/clone_materializer/chained.rs b/nodedb/src/control/maintenance/clone_materializer/chained.rs index 30758a4ca..c4f942e1c 100644 --- a/nodedb/src/control/maintenance/clone_materializer/chained.rs +++ b/nodedb/src/control/maintenance/clone_materializer/chained.rs @@ -265,6 +265,7 @@ async fn commit_relink( target.vshard_id, payload, ); - propose_replicated_entry(state, proposer, entry).await?; + let deadline = crate::control::wal_replication::statement_propose_deadline(state); + propose_replicated_entry(state, proposer, entry, deadline).await?; Ok(()) } diff --git a/nodedb/src/control/metadata_proposer/handle.rs b/nodedb/src/control/metadata_proposer/handle.rs index bf060605e..175799028 100644 --- a/nodedb/src/control/metadata_proposer/handle.rs +++ b/nodedb/src/control/metadata_proposer/handle.rs @@ -136,6 +136,7 @@ fn metadata_propose_error(error: ClusterError) -> Error { | ClusterError::GhostNotFound { .. } | ClusterError::Transport { .. } | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. } | ClusterError::Storage { .. } | ClusterError::Codec { .. } | ClusterError::UnsupportedWireVersion { .. } diff --git a/nodedb/src/control/orchestrated_write.rs b/nodedb/src/control/orchestrated_write.rs index e890d48a5..3cf4c82a3 100644 --- a/nodedb/src/control/orchestrated_write.rs +++ b/nodedb/src/control/orchestrated_write.rs @@ -49,7 +49,8 @@ pub(crate) async fn apply_orchestrated_write( } })?; let request_id = RequestId::new(state.request_id_counter.fetch_add(1, Ordering::Relaxed)); - match propose_replicated_entry(state, proposer, entry).await { + let deadline = crate::control::wal_replication::statement_propose_deadline(state); + match propose_replicated_entry(state, proposer, entry, deadline).await { Ok((payload, write_version)) => { let response = Response { request_id, diff --git a/nodedb/src/control/planner/calvin/reservation.rs b/nodedb/src/control/planner/calvin/reservation.rs index 191d01ed4..e37c6b174 100644 --- a/nodedb/src/control/planner/calvin/reservation.rs +++ b/nodedb/src/control/planner/calvin/reservation.rs @@ -22,8 +22,8 @@ use std::time::Duration; use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; use nodedb_cluster::calvin::types::{LockKeyWire, ReleaseReason, TxnIdWire}; use nodedb_cluster::{ - RaftRpc, ReleaseReservationRequest, ReleaseReservationResponse, ReserveReadRequest, - ReserveReadResponse, + ClusterError, RaftRpc, ReleaseReservationRequest, ReleaseReservationResponse, + ReserveReadRequest, ReserveReadResponse, }; use crate::Error; @@ -86,8 +86,6 @@ pub(crate) async fn submit_reserve_read( vshard: u32, owner: Option, ) -> crate::Result { - let local_timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); - // Every running server has a cluster transport, the synthesized one-node // cluster included. Without one, `start_raft` never ran here. let Some(transport) = state.cluster_transport.as_ref() else { @@ -116,6 +114,7 @@ pub(crate) async fn submit_reserve_read( // Leader is self: submit locally (a self-RPC will be a pointless extra // hop and the local inbox is the one that gets the assignment). if leader == state.node_id { + let local_timeout = super::submit::budget::statement_budget(state)?; return submit_local_reserve_read(state, key, vshard, owner, local_timeout).await; } @@ -138,12 +137,8 @@ pub(crate) async fn submit_reserve_read( }) .transpose()?; - let deadline_remaining_ms = state - .tuning - .network - .default_deadline_secs - .saturating_mul(1000) - .max(1); + // The leader works on the reservation only as long as the statement waits. + let deadline_remaining_ms = super::submit::budget::statement_budget_ms(state)?; let req = ReserveReadRequest { lock_key_bytes, vshard, @@ -176,6 +171,11 @@ pub(crate) async fn submit_reserve_read( Ok(other) => Err(Error::Internal { detail: format!("reserve-read: unexpected reply from node {leader}: {other:?}"), }), + // The request went out and its answer was lost. The leader can have + // granted it, so its outcome is unknown. Lease GC reaps a lost grant. + Err(ClusterError::Unanswered { .. }) => Err(Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + }), Err(e) => Err(Error::Internal { detail: format!("reserve-read RPC to sequencer leader node {leader} failed: {e}"), }), @@ -282,6 +282,9 @@ pub(crate) async fn release_reservation( } }; + // A release runs on the abort path too, often after the statement's + // deadline passed. It frees a lock other statements wait on, so it takes + // the node default, never what is left of the statement. let deadline_remaining_ms = state .tuning .network diff --git a/nodedb/src/control/planner/calvin/submit/assign.rs b/nodedb/src/control/planner/calvin/submit/assign.rs index 4919bb63d..a4a5bca27 100644 --- a/nodedb/src/control/planner/calvin/submit/assign.rs +++ b/nodedb/src/control/planner/calvin/submit/assign.rs @@ -10,7 +10,7 @@ use std::time::Duration; use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; use nodedb_cluster::calvin::types::TxClass; -use nodedb_cluster::{RaftRpc, SubmitCalvinInboxRequest, SubmitCalvinInboxResponse}; +use nodedb_cluster::{ClusterError, RaftRpc, SubmitCalvinInboxRequest, SubmitCalvinInboxResponse}; use super::stream::{PartStream, StreamTarget, stream_parts}; use crate::Error; @@ -122,7 +122,6 @@ pub async fn submit_calvin_routed_assign( super::local::stamp_incarnations(state, &mut tx_class)?; super::unique_claims::stamp_unique_claims(state, &mut tx_class)?; let stream = super::parts::split_into_parts(state, &mut tx_class)?; - let local_timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); // Every running server has a cluster transport, the synthesized one-node // cluster included. Without one, `start_raft` never ran here. @@ -153,6 +152,7 @@ pub async fn submit_calvin_routed_assign( // Leader is self: submit-and-assign locally (a self-RPC will be a pointless // extra hop and the local registry is the one that gets the assignment). if leader == state.node_id { + let local_timeout = super::budget::statement_budget(state)?; return assign_prepared(state, tx_class, stream, local_timeout).await; } @@ -167,12 +167,8 @@ pub async fn submit_calvin_routed_assign( detail: format!("failed to encode TxClass for routed Calvin inbox submit: {e}"), })?; - let deadline_remaining_ms = state - .tuning - .network - .default_deadline_secs - .saturating_mul(1000) - .max(1); + // The leader works on the transaction only as long as the statement waits. + let deadline_remaining_ms = super::budget::statement_budget_ms(state)?; let req = SubmitCalvinInboxRequest { tx_class_bytes, deadline_remaining_ms, @@ -213,6 +209,13 @@ pub async fn submit_calvin_routed_assign( detail: format!("calvin-inbox: unexpected reply from node {leader}: {other:?}"), }); } + // The submit went out and its answer was lost. The leader can have + // assigned it, so its outcome is unknown. + Err(ClusterError::Unanswered { .. }) => { + return Err(Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + }); + } Err(e) => { return Err(Error::Internal { detail: format!("calvin-inbox RPC to sequencer leader node {leader} failed: {e}"), diff --git a/nodedb/src/control/planner/calvin/submit/budget.rs b/nodedb/src/control/planner/calvin/submit/budget.rs new file mode 100644 index 000000000..7d50b8dae --- /dev/null +++ b/nodedb/src/control/planner/calvin/submit/budget.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The budget a Calvin request carries to the sequencer leader. +//! +//! A Calvin submit belongs to the statement that issued it. The sequencer +//! leader works on it only as long as the statement waits. The budget is +//! what is left of the statement's deadline, never the node default. + +use std::time::Duration; + +use crate::bridge::envelope::ErrorCode; +use crate::control::state::SharedState; +use crate::control::wal_replication::statement_propose_deadline; + +/// The milliseconds left to `deadline`, for a request about to go out. +/// +/// A spent budget refuses the request before it goes out. Nothing was +/// submitted, so the refusal is definite. +pub(crate) fn budget_ms_until(deadline: tokio::time::Instant) -> crate::Result { + match nodedb_cluster::rpc_codec::remaining_budget_ms(deadline) { + 0 => Err(crate::Error::DataPlane(ErrorCode::ExpiredBeforeExecution)), + ms => Ok(ms), + } +} + +/// The milliseconds left on the running statement, or on the node default +/// outside a statement. +pub(crate) fn statement_budget_ms(state: &SharedState) -> crate::Result { + budget_ms_until(statement_propose_deadline(state)) +} + +/// [`statement_budget_ms`] as a wait bound for the local half of the request. +pub(crate) fn statement_budget(state: &SharedState) -> crate::Result { + statement_budget_ms(state).map(Duration::from_millis) +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::control::server::shared::session::{conn_scope, deadline, statement_deadline}; + + /// A statement with 100 ms left forwards at most 100 ms, under a node + /// default of 30 s. + #[tokio::test] + async fn the_budget_ends_within_the_statement_deadline() { + conn_scope::scoped(async { + let _statement = deadline::enter(Some(Duration::from_millis(100)), 30); + let at = tokio::time::Instant::from_std(statement_deadline(30)); + let budget = budget_ms_until(at).expect("a live budget"); + assert!( + budget <= 100, + "budget {budget} ms exceeds the statement's 100 ms" + ); + assert!(budget > 0); + }) + .await; + } + + /// A spent statement sends nothing. Its refusal is definite. + #[tokio::test] + async fn a_spent_statement_is_refused_before_it_goes_out() { + let error = + budget_ms_until(tokio::time::Instant::now()).expect_err("a spent budget is refused"); + let crate::Error::DataPlane(code) = &error else { + panic!("expected a Data-Plane verdict, got {error:?}"); + }; + assert_eq!(code, &ErrorCode::ExpiredBeforeExecution); + assert!(crate::control::server::dispatch_utils::write_definitely_not_applied(code)); + } +} diff --git a/nodedb/src/control/planner/calvin/submit/mod.rs b/nodedb/src/control/planner/calvin/submit/mod.rs index ee43da39d..20bec2a59 100644 --- a/nodedb/src/control/planner/calvin/submit/mod.rs +++ b/nodedb/src/control/planner/calvin/submit/mod.rs @@ -31,6 +31,7 @@ //! schedulers; this module never does storage I/O or io_uring directly. pub mod assign; +pub mod budget; pub mod edge_slices; pub mod local; pub mod parts; diff --git a/nodedb/src/control/planner/calvin/submit/routed.rs b/nodedb/src/control/planner/calvin/submit/routed.rs index 07508790f..45bfdac59 100644 --- a/nodedb/src/control/planner/calvin/submit/routed.rs +++ b/nodedb/src/control/planner/calvin/submit/routed.rs @@ -11,7 +11,9 @@ use std::time::Duration; use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; use nodedb_cluster::calvin::types::TxClass; -use nodedb_cluster::{RaftRpc, SubmitCalvinTxnRequest, SubmitCalvinTxnResponse, TypedClusterError}; +use nodedb_cluster::{ + ClusterError, RaftRpc, SubmitCalvinTxnRequest, SubmitCalvinTxnResponse, TypedClusterError, +}; use crate::Error; use crate::bridge::envelope::Response; @@ -120,7 +122,7 @@ pub async fn submit_calvin_routed( // Leader is self: submit-and-await locally (a self-RPC will be a pointless // extra hop and the local registry is the one that completes). if leader == state.node_id { - let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); + let timeout = super::budget::statement_budget(state)?; return submit_prepared_and_await(state, tx_class, stream, timeout).await; } @@ -135,12 +137,8 @@ pub async fn submit_calvin_routed( detail: format!("failed to encode TxClass for routed Calvin submit: {e}"), })?; - let deadline_remaining_ms = state - .tuning - .network - .default_deadline_secs - .saturating_mul(1000) - .max(1); + // The leader works on the transaction only as long as the statement waits. + let deadline_remaining_ms = super::budget::statement_budget_ms(state)?; let req = SubmitCalvinTxnRequest { tx_class_bytes, deadline_remaining_ms, @@ -204,6 +202,11 @@ pub async fn submit_calvin_routed( Ok(other) => Err(Error::Internal { detail: format!("calvin-submit: unexpected reply from node {leader}: {other:?}"), }), + // The submit went out and its answer was lost. The leader can have + // sequenced it, so its outcome is unknown. + Err(ClusterError::Unanswered { .. }) => Err(Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + }), Err(e) => Err(Error::Internal { detail: format!("calvin-submit RPC to sequencer leader node {leader} failed: {e}"), }), diff --git a/nodedb/src/control/planner/calvin/submit/unique_claims.rs b/nodedb/src/control/planner/calvin/submit/unique_claims.rs index f10092dfd..c49d12ee9 100644 --- a/nodedb/src/control/planner/calvin/submit/unique_claims.rs +++ b/nodedb/src/control/planner/calvin/submit/unique_claims.rs @@ -55,10 +55,31 @@ pub(crate) fn stamp_unique_claims( detail: format!("calvin unique claims: plan decode: {e}"), } })?; + let sets = catalog_unique_claim_sets( + state, + tx_class.database_id, + tx_class.tenant_id.as_u64(), + &plans, + )?; + for set in sets { + if !tx_class.write_set.0.contains(&set) { + tx_class.write_set.0.push(set); + } + } + Ok(()) +} + +/// The UNIQUE claim key sets of `plans`, with each collection's UNIQUE +/// indexes read from this node's catalog. A Calvin transaction and an +/// autocommit write at the write-admission gate both claim through it. +pub(crate) fn catalog_unique_claim_sets( + state: &SharedState, + database_id: nodedb_types::DatabaseId, + tenant_id: u64, + plans: &[PhysicalPlan], +) -> crate::Result> { let catalog = state.credentials.catalog(); - let database_id = tx_class.database_id; - let tenant_id = tx_class.tenant_id.as_u64(); - let sets = unique_claim_sets(&plans, |collection| { + unique_claim_sets(plans, |collection| { let key = nodedb_types::CollectionKey::from_qualified_str(database_id, collection) .unwrap_or_else(|_| nodedb_types::CollectionKey::from_bare(database_id, collection)); let stored = catalog.get_collection(key.database_id(), tenant_id, key.name())?; @@ -79,13 +100,7 @@ pub(crate) fn stamp_unique_claims( }) .collect() })) - })?; - for set in sets { - if !tx_class.write_set.0.contains(&set) { - tx_class.write_set.0.push(set); - } - } - Ok(()) + }) } /// The UNIQUE claim key sets of `plans`. `unique_paths` names the UNIQUE diff --git a/nodedb/src/control/server/dispatch_utils/durable_write.rs b/nodedb/src/control/server/dispatch_utils/durable_write.rs index da5b51143..5229ffbd0 100644 --- a/nodedb/src/control/server/dispatch_utils/durable_write.rs +++ b/nodedb/src/control/server/dispatch_utils/durable_write.rs @@ -215,7 +215,9 @@ async fn propose_if_replicable( return Ok(None); }; let entry = entry.with_event_source(event_source); - let (payload, write_version) = propose_replicated_entry(shared, proposer, entry).await?; + let deadline = crate::control::wal_replication::statement_propose_deadline(shared); + let (payload, write_version) = + propose_replicated_entry(shared, proposer, entry, deadline).await?; Ok(Some(replicated_write_response( shared, payload, diff --git a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/admission.rs b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/admission.rs index 7315b095e..8203dac78 100644 --- a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/admission.rs +++ b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/admission.rs @@ -5,14 +5,27 @@ use tokio::sync::OwnedMutexGuard; use crate::bridge::envelope::PhysicalPlan; +use crate::control::server::shared::write_admission::gate::admit_routed; use crate::control::server::shared::write_admission::{ - WriteAdmission, WriteAdmissionGuard, WriteTarget, admit, + WriteAdmission, WriteAdmissionGuard, WriteTarget, }; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId, VShardId}; use super::super::params::WriteOrdering; +/// The write the admission phase admits. +pub(super) struct AdmissionTarget<'a> { + pub tenant_id: TenantId, + pub database_id: DatabaseId, + pub vshard_id: VShardId, + pub plan: &'a PhysicalPlan, + /// Whether a contended write may run through the scheduler, which + /// applies it as a client write. A write of another event source waits + /// for its keys instead. + pub may_route: bool, +} + /// Outcome of the write-admission phase. pub(super) enum AdmissionOutcome { /// Proceed with the funnel's remaining phases under these guards. @@ -26,25 +39,33 @@ pub(super) enum AdmissionOutcome { } /// Write-admission gate: every write-class plan whose ordering is not already -/// final passes here. An uncontended point write takes the fast path holding -/// its per-vShard deterministic locks; a contended or bulk write is submitted -/// through the deterministic scheduler and its applied response is surfaced -/// here; reads / control ops are `Exempt`. +/// final passes here. An uncontended write takes the fast path holding its +/// per-vShard deterministic locks; a contended write is submitted through the +/// deterministic scheduler and its applied response is surfaced here; reads / +/// control ops are `Exempt`. /// /// Ordering (fast path): the guard is acquired FIRST, then — for a write that /// owns its durability (`AppendHere`) — the WAL append happens after this /// call returns, under the guard, minting the LSN just before the enqueue. /// The guard is released immediately after the enqueue (not across the /// response await). +/// +/// A contended write no Calvin transaction can sequence waits for its keys +/// until `deadline`, holding none meanwhile. pub(super) async fn admit_write( shared: &SharedState, - tenant_id: TenantId, - database_id: DatabaseId, - vshard_id: VShardId, - plan: &PhysicalPlan, + target: AdmissionTarget<'_>, ordering: WriteOrdering, -) -> AdmissionOutcome { - match ordering { + deadline: std::time::Instant, +) -> crate::Result { + let AdmissionTarget { + tenant_id, + database_id, + vshard_id, + plan, + may_route, + } = target; + Ok(match ordering { WriteOrdering::AlreadyOrdered => AdmissionOutcome::Proceed { admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, @@ -52,7 +73,7 @@ pub(super) async fn admit_write( admission_guard: None, order_guard: None, }, - WriteOrdering::Gate => match admit( + WriteOrdering::Gate => match admit_routed( shared, &WriteTarget { tenant_id, @@ -60,7 +81,8 @@ pub(super) async fn admit_write( vshard_id, plan, }, - ) { + may_route, + )? { WriteAdmission::ExemptRead => AdmissionOutcome::Proceed { admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, @@ -89,6 +111,14 @@ pub(super) async fn admit_write( } } WriteAdmission::RouteToCalvin => AdmissionOutcome::RouteToCalvin, + WriteAdmission::Wait(wait) => AdmissionOutcome::Proceed { + admission: crate::bridge::envelope::Admission::Admitted, + admission_guard: Some( + wait.acquire(tokio::time::Instant::from_std(deadline)) + .await?, + ), + order_guard: None, + }, }, - } + }) } diff --git a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/driver.rs b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/driver.rs index 8a4b92400..4c7482709 100644 --- a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/driver.rs +++ b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/driver.rs @@ -16,7 +16,7 @@ use crate::control::server::dispatch_utils::minted::{ }; use crate::control::server::shared::session::statement_deadline; use crate::control::server::shared::write_admission::{ - bare_ok_response, order_row_write, route_write_to_calvin, + bare_ok_response, calvin_route_keeps, order_row_write, route_write_to_calvin, }; use crate::control::server::wal_dispatch; use crate::control::state::SharedState; @@ -25,7 +25,7 @@ use crate::engine::timeseries::resolved_ingest::TsDriftPolicy; use super::super::params::{ ChangeFeedOwner, SubmitOutcome, SubmitWrite, WalDurability, WriteOrdering, }; -use super::admission::{AdmissionOutcome, admit_write}; +use super::admission::{AdmissionOutcome, AdmissionTarget, admit_write}; use super::answer::Answer; use super::dispatch::{DispatchTarget, HeldGuards, dispatch_to_data_plane}; use super::late_parts::LateParts; @@ -201,32 +201,55 @@ pub(crate) async fn enqueue_write( // TxClass plus its own `CalvinApplied` WAL record), so no local WAL append or // enqueue happens. A plain write with no RETURNING rows yields `None`, // synthesized into a bare `Ok`. - let (admission, admission_guard, order_guard) = - match admit_write(shared, tenant_id, database_id, vshard_id, &plan, ordering).await { - AdmissionOutcome::Proceed { - admission, - admission_guard, - order_guard, - } => (admission, admission_guard, order_guard), - AdmissionOutcome::RouteToCalvin => { - // The scheduler journals the write in its own sequenced record. - // The scheduler applies the write from its own records, so - // the caller's records never apply. - let superseded = caller_minted.map(|minted| { - minted.supersede(std::sync::Arc::clone(&shared.wal), owner, "calvin_route") - }); - let routed = - route_write_to_calvin(shared, tenant_id, database_id, vshard_id, plan).await; - if let Some(superseded) = superseded { - superseded.finish().await; - } - let routed = routed?; - return Ok(PendingWrite::done(SubmitOutcome { - response: routed - .unwrap_or_else(|| bare_ok_response(crate::types::RequestId::new(0))), - })); + let target = AdmissionTarget { + tenant_id, + database_id, + vshard_id, + plan: &plan, + may_route: calvin_route_keeps(&plan, event_source, 0), + }; + let admitted = match admit_write(shared, target, ordering, deadline).await { + Ok(admitted) => admitted, + Err(error) => { + // No record of this write reaches a core. + if let Some(minted) = caller_minted { + minted.cancel(&shared.wal, owner, 0).await?; } - }; + return Err(error); + } + }; + let (admission, admission_guard, order_guard) = match admitted { + AdmissionOutcome::Proceed { + admission, + admission_guard, + order_guard, + } => (admission, admission_guard, order_guard), + AdmissionOutcome::RouteToCalvin => { + // The scheduler journals the write in its own sequenced record. + // The scheduler applies the write from its own records, so + // the caller's records never apply. + let superseded = caller_minted.map(|minted| { + minted.supersede(std::sync::Arc::clone(&shared.wal), owner, "calvin_route") + }); + let routed = route_write_to_calvin( + shared, + tenant_id, + database_id, + vshard_id, + plan, + event_source, + ) + .await; + if let Some(superseded) = superseded { + superseded.finish().await; + } + let routed = routed?; + return Ok(PendingWrite::done(SubmitOutcome { + response: routed + .unwrap_or_else(|| bare_ok_response(crate::types::RequestId::new(0))), + })); + } + }; // Order this write's row records (document rows and edges) against every // other row write of its vShard, from before the append below through its diff --git a/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs b/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs index 6741589f7..7fb9e7638 100644 --- a/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs +++ b/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs @@ -30,9 +30,13 @@ impl NodeDbPgHandler { // `write_version` is the post-write `coll_write_lsn`, surfaced so the session // can floor a later read-set at it (read-your-writes for cross-shard OCC). - let (payload, write_version) = - crate::control::wal_replication::propose_replicated_entry(&self.state, proposer, entry) - .await?; + let (payload, write_version) = crate::control::wal_replication::propose_replicated_entry( + &self.state, + proposer, + entry, + crate::control::wal_replication::statement_propose_deadline(&self.state), + ) + .await?; let response = Response { request_id, diff --git a/nodedb/src/control/server/shared/session/commit/single_shard.rs b/nodedb/src/control/server/shared/session/commit/single_shard.rs index 2ea89a2a4..6d3fee1d1 100644 --- a/nodedb/src/control/server/shared/session/commit/single_shard.rs +++ b/nodedb/src/control/server/shared/session/commit/single_shard.rs @@ -213,7 +213,8 @@ async fn commit_redo( target.vshard_id, payload, ); - match propose_replicated_entry(state, proposer, entry).await { + let deadline = crate::control::wal_replication::statement_propose_deadline(state); + match propose_replicated_entry(state, proposer, entry, deadline).await { Ok((applied, _)) => Ok(super::ts_rejections::applied_rejected_by_collection( &applied, )), diff --git a/nodedb/src/control/server/shared/write_admission/admission_keys.rs b/nodedb/src/control/server/shared/write_admission/admission_keys.rs new file mode 100644 index 000000000..768d6424f --- /dev/null +++ b/nodedb/src/control/server/shared/write_admission/admission_keys.rs @@ -0,0 +1,524 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The lock keys the write-admission gate takes for one write. +//! +//! The keys come from the Calvin write-key extraction and its lock-request +//! expansion. An autocommit write therefore takes the keys a Calvin +//! transaction of the same plan takes: +//! +//! - A row write locks each row `Exclusive` and its collection `Intent`. +//! - A collection-wide write locks its collection `Exclusive`: a predicate +//! or bulk write, INSERT…SELECT, a truncate, a columnar update or delete, +//! an array write, a KV predicate write, and a CRDT snapshot import. +//! - A document write that claims a UNIQUE value locks that value +//! `Exclusive`. +//! +//! An append differs from its Calvin keys: a timeseries ingest or a columnar +//! insert locks the rows it names `Exclusive` and its collection `Intent`. +//! +//! - Two appends never contend at the gate. The data-group log orders them, +//! alike on every replica. +//! - Every Calvin writer of a columnar or timeseries collection locks it +//! `Exclusive`. So an append still orders against each Calvin writer. +//! - A contended append waits and never routes. A routed write applies +//! outside the data-group log. Each replica then installs it against its +//! own schema at its own point. +//! +//! A write no Calvin transaction sequences still takes keys: +//! +//! - A resolved write locks the rows it names, as a row write does. +//! - A text or spatial index write locks its collection `Intent`. +//! - A transaction batch takes the keys of every plan it carries. +//! - Any other write locks every collection it writes `Exclusive`. + +use std::collections::BTreeMap; + +use nodedb_cluster::calvin::types::EngineKeySet; +use nodedb_physical::physical_plan::{ + ColumnarOp, DocumentOp, KvOp, MetaOp, PhysicalPlan, TimeseriesOp, VectorOp, +}; + +use crate::control::cluster::calvin::scheduler::driver::helpers::expand_write_key_sets; +use crate::control::cluster::calvin::scheduler::lock_manager::{LockKey, LockMode}; +use crate::control::planner::calvin::submit::unique_claims::catalog_unique_claim_sets; +use crate::control::planner::calvin::tx_class::write_keys::{WriteKeys, add_plan_write_keys}; +use crate::control::planner::calvin::write_class::is_write_plan; +use crate::control::state::SharedState; +use crate::control::wal_replication::transaction_redo::collections::written_collections; +use crate::types::{DatabaseId, TenantId}; + +/// The moded lock request of one write. +#[derive(Debug, Default)] +pub(crate) struct AdmissionKeys { + /// Every key the write holds, each in its mode. + pub keys: BTreeMap, + /// Whether a contended write routes to the scheduler. A write a Calvin + /// transaction cannot sequence, and an append, waits instead. + pub sequenced: bool, +} + +/// The lock request of `plan`, a write-class plan of `tenant_id` in +/// `database_id` that no Calvin scheduler applies. +pub(crate) fn plan_admission_keys( + shared: &SharedState, + tenant_id: TenantId, + database_id: DatabaseId, + plan: &PhysicalPlan, +) -> crate::Result { + let (mut sets, sequenced) = plan_key_sets(plan)?; + if sequenced { + // A claim the plan does not carry locks the collection whole. The + // expansion merges each key's modes, so the strongest mode wins. + sets.extend(catalog_unique_claim_sets( + shared, + database_id, + tenant_id.as_u64(), + std::slice::from_ref(plan), + )?); + } + Ok(AdmissionKeys { + keys: expand_write_key_sets(&sets), + sequenced, + }) +} + +/// The lock request of a write that names only whole `collections`, each +/// locked `Exclusive`. A committed session transaction's redo and a synced +/// array op take it. +pub(crate) fn collection_admission_keys(collections: &[String]) -> AdmissionKeys { + let mut keys = WriteKeys::default(); + for collection in collections { + keys.whole_collection(collection); + } + AdmissionKeys { + keys: expand_write_key_sets(&keys.into_key_sets()), + sequenced: false, + } +} + +/// A Calvin-scheduled apply: the scheduler acquired its locks. +pub(super) fn is_calvin_apply(plan: &PhysicalPlan) -> bool { + matches!( + plan, + PhysicalPlan::Meta( + MetaOp::CalvinExecuteStatic { .. } + | MetaOp::CalvinExecutePassive { .. } + | MetaOp::CalvinExecuteActive { .. } + | MetaOp::RecordCalvinWriteVersions { .. } + | MetaOp::CalvinFlush { .. } + | MetaOp::CalvinDrop { .. } + | MetaOp::CalvinResolve { .. } + ) + ) +} + +/// The write key sets of `plan`, and whether it routes to the scheduler +/// when contended. +fn plan_key_sets(plan: &PhysicalPlan) -> crate::Result<(Vec, bool)> { + let mut keys = WriteKeys::default(); + match plan { + // An append: its rows, and its collection `Intent` (see the module + // docs). A timeseries row has no surrogate: its series names it. + PhysicalPlan::Timeseries(TimeseriesOp::Ingest { collection, .. }) => { + keys.rows(collection.as_str(), []); + } + PhysicalPlan::Columnar(ColumnarOp::Insert { + collection, + surrogates, + .. + }) => keys.rows( + collection.as_str(), + surrogates.iter().map(|surrogate| surrogate.as_u32()), + ), + PhysicalPlan::Meta(MetaOp::TransactionBatch { plans, .. }) => { + let mut sets = Vec::new(); + for inner in plans.iter().filter(|inner| is_write_plan(inner)) { + sets.extend(plan_key_sets(inner)?.0); + } + return Ok((sets, false)); + } + // An index write of rows its collection stores. No Calvin + // transaction writes the index alone, so the collection key orders + // it against a collection-wide writer. + PhysicalPlan::Text(_) | PhysicalPlan::Spatial(_) => { + for collection in written_collections(std::slice::from_ref(plan)) { + keys.rows(&collection, []); + } + } + PhysicalPlan::Document(_) + | PhysicalPlan::Kv(_) + | PhysicalPlan::Vector(_) + | PhysicalPlan::Graph(_) + | PhysicalPlan::Timeseries(_) + | PhysicalPlan::Columnar(_) + | PhysicalPlan::Crdt(_) + | PhysicalPlan::Array(_) + | PhysicalPlan::Meta(_) + | PhysicalPlan::Query(_) + | PhysicalPlan::ClusterArray(_) + | PhysicalPlan::ClusterEvent(_) => { + if is_write_plan(plan) && add_plan_write_keys(&mut keys, plan).is_ok() { + return Ok((keys.into_key_sets(), true)); + } + // The extraction refuses a write no Calvin transaction can + // sequence. The keys it added before refusing stay: they + // over-lock and never under-lock. + unsequenced_keys(&mut keys, plan); + } + } + Ok((keys.into_key_sets(), false)) +} + +/// Add the keys of a write no Calvin transaction sequences: the rows a +/// resolved write names, or every collection the write names, whole. +fn unsequenced_keys(keys: &mut WriteKeys, plan: &PhysicalPlan) { + match plan { + PhysicalPlan::Document(DocumentOp::ResolvedWrite { mutations, .. }) => { + for mutation in mutations { + let collection = mutation.collection().as_str(); + keys.rows(collection, [mutation.surrogate().as_u32()]); + keys.row_id(collection, mutation.document_id()); + } + } + PhysicalPlan::Kv(KvOp::ResolvedWrite { mutations, .. }) => { + for mutation in mutations { + keys.kv_keys(mutation.collection().as_str(), [mutation.key().to_vec()]); + } + } + PhysicalPlan::Vector(VectorOp::ResolvedDirectWrite { + collection, + mutations, + .. + }) => keys.vector_rows( + collection.as_str(), + mutations + .iter() + .map(|mutation| mutation.surrogate().as_u32()), + ), + PhysicalPlan::Document(_) + | PhysicalPlan::Kv(_) + | PhysicalPlan::Vector(_) + | PhysicalPlan::Graph(_) + | PhysicalPlan::Timeseries(_) + | PhysicalPlan::Columnar(_) + | PhysicalPlan::Crdt(_) + | PhysicalPlan::Array(_) + | PhysicalPlan::Meta(_) + | PhysicalPlan::Text(_) + | PhysicalPlan::Spatial(_) + | PhysicalPlan::Query(_) + | PhysicalPlan::ClusterArray(_) + | PhysicalPlan::ClusterEvent(_) => { + for collection in written_collections(std::slice::from_ref(plan)) { + keys.whole_collection(&collection); + } + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use nodedb_physical::physical_plan::{ColumnarInsertIntent, CrdtOp}; + use nodedb_physical::physical_plan::{DocumentResolvedMutation, KvResolvedMutation}; + + use crate::control::cluster::calvin::scheduler::lock_manager::{LockManager, TxnId}; + use nodedb_types::{QualifiedCollection, Surrogate}; + + use super::*; + + fn qualified(name: &str) -> QualifiedCollection { + QualifiedCollection::new(DatabaseId::DEFAULT, name) + } + + fn coll(name: &str) -> LockKey { + LockKey::Collection { + collection: Arc::from(qualified(name).as_str()), + } + } + + fn surrogate_key(name: &str, surrogate: u32) -> LockKey { + LockKey::Surrogate { + collection: Arc::from(qualified(name).as_str()), + surrogate, + } + } + + fn kv_key(name: &str, key: &[u8]) -> LockKey { + LockKey::Kv { + collection: Arc::from(qualified(name).as_str()), + key: Arc::from(key), + } + } + + /// The keys of `plan`, with no UNIQUE claim lookup. + fn keys_of(plan: &PhysicalPlan) -> (BTreeMap, bool) { + let (sets, sequenced) = plan_key_sets(plan).expect("key sets"); + (expand_write_key_sets(&sets), sequenced) + } + + fn doc_insert(id: &str, surrogate: u32) -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::PointInsert { + collection: qualified("docs"), + document_id: id.to_owned(), + value: Vec::new(), + if_absent: false, + surrogate: Surrogate::new(surrogate), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: Vec::new(), + }) + } + + /// A point write locks its row `Exclusive`, by surrogate and by row id, + /// and its collection `Intent`. + #[test] + fn a_point_write_locks_its_row_and_announces_its_collection() { + let (keys, sequenced) = keys_of(&doc_insert("x", 17)); + assert!(sequenced); + assert_eq!(keys.get(&coll("docs")), Some(&LockMode::Intent)); + assert_eq!( + keys.get(&surrogate_key("docs", 17)), + Some(&LockMode::Exclusive) + ); + assert_eq!(keys.get(&kv_key("docs", b"x")), Some(&LockMode::Exclusive)); + assert_eq!(keys.len(), 3); + } + + /// A batch insert locks every row it inserts and its collection + /// `Intent`. + #[test] + fn a_batch_insert_locks_each_row() { + let plan = PhysicalPlan::Document(DocumentOp::BatchInsert { + collection: qualified("docs"), + documents: vec![("a".to_owned(), Vec::new()), ("b".to_owned(), Vec::new())], + surrogates: vec![Surrogate::new(1), Surrogate::new(2)], + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: Vec::new(), + }); + let (keys, sequenced) = keys_of(&plan); + assert!(sequenced); + assert_eq!(keys.get(&coll("docs")), Some(&LockMode::Intent)); + for (id, surrogate) in [("a", 1), ("b", 2)] { + assert_eq!( + keys.get(&surrogate_key("docs", surrogate)), + Some(&LockMode::Exclusive) + ); + assert_eq!( + keys.get(&kv_key("docs", id.as_bytes())), + Some(&LockMode::Exclusive) + ); + } + } + + /// Every collection-wide shape locks its collection `Exclusive` and + /// routes to the scheduler when contended. + #[test] + fn a_collection_wide_write_locks_its_collection_exclusive() { + let plans = [ + PhysicalPlan::Document(DocumentOp::Truncate { + collection: qualified("docs"), + restart_identity: false, + resolved_sum_targets: Vec::new(), + declared_primary_key: None, + }), + PhysicalPlan::Columnar(ColumnarOp::Delete { + collection: qualified("docs"), + filters: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::pending_injection(), + }), + PhysicalPlan::Timeseries(TimeseriesOp::Truncate { + collection: qualified("docs"), + restart_identity: false, + }), + PhysicalPlan::Kv(KvOp::Truncate { + collection: qualified("docs"), + restart_identity: false, + }), + PhysicalPlan::Crdt(CrdtOp::ImportSnapshot { + tenant_id: 1, + collection: qualified("docs"), + bytes: Vec::new(), + }), + ]; + for plan in plans { + let (keys, sequenced) = keys_of(&plan); + assert!(sequenced, "{plan:?}"); + assert_eq!( + keys.get(&coll("docs")), + Some(&LockMode::Exclusive), + "{plan:?}" + ); + } + } + + /// A resolved write no Calvin transaction sequences still locks the rows + /// it names, and waits rather than routes when contended. + #[test] + fn a_resolved_write_locks_the_rows_it_names() { + let doc = PhysicalPlan::Document(DocumentOp::ResolvedWrite { + mutations: vec![DocumentResolvedMutation::Delete { + collection: qualified("docs"), + document_id: "x".to_owned(), + surrogate: Surrogate::new(17), + pk_bytes: b"x".to_vec(), + precondition: None, + resolved_sum_targets: Vec::new(), + }], + response_payload: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::pending_injection(), + }); + let (keys, sequenced) = keys_of(&doc); + assert!(!sequenced); + assert_eq!( + keys.get(&surrogate_key("docs", 17)), + Some(&LockMode::Exclusive) + ); + assert_eq!(keys.get(&coll("docs")), Some(&LockMode::Intent)); + + let kv = PhysicalPlan::Kv(KvOp::ResolvedWrite { + mutations: vec![KvResolvedMutation::Put { + collection: qualified("kv"), + key: b"k".to_vec(), + value: Vec::new(), + ttl_ms: 0, + expire_at_ms: 0, + surrogate: Surrogate::new(5), + precondition: None, + }], + response_payload: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::pending_injection(), + }); + let (keys, sequenced) = keys_of(&kv); + assert!(!sequenced); + assert_eq!(keys.get(&kv_key("kv", b"k")), Some(&LockMode::Exclusive)); + assert_eq!(keys.get(&coll("kv")), Some(&LockMode::Intent)); + } + + /// A transaction batch takes the keys of every write it carries. + #[test] + fn a_transaction_batch_takes_the_keys_of_its_writes() { + let batch = PhysicalPlan::Meta(MetaOp::TransactionBatch { + plans: vec![doc_insert("x", 17), doc_insert("y", 18)], + txn_id: None, + }); + let (keys, sequenced) = keys_of(&batch); + assert!(!sequenced); + assert_eq!( + keys.get(&surrogate_key("docs", 17)), + Some(&LockMode::Exclusive) + ); + assert_eq!( + keys.get(&surrogate_key("docs", 18)), + Some(&LockMode::Exclusive) + ); + assert_eq!(keys.get(&coll("docs")), Some(&LockMode::Intent)); + } + + fn ts_ingest() -> PhysicalPlan { + PhysicalPlan::Timeseries(TimeseriesOp::Ingest { + collection: qualified("metrics"), + payload: b"metrics value=1.0 1000000000".to_vec(), + format: "ilp".to_owned(), + wal_lsn: None, + surrogates: Vec::new(), + provenance: None, + rls_write_check: nodedb_types::RlsWriteCheck::pending_injection(), + returning: None, + rls_filters: Vec::new(), + }) + } + + fn columnar_insert(intent: ColumnarInsertIntent) -> PhysicalPlan { + PhysicalPlan::Columnar(ColumnarOp::Insert { + collection: qualified("events"), + payload: Vec::new(), + format: "msgpack".to_owned(), + intent, + on_conflict_updates: Vec::new(), + surrogates: vec![Surrogate::new(4), Surrogate::new(5)], + schema_bytes: Vec::new(), + provenance: None, + wal_lsn: None, + rls_write_check: nodedb_types::RlsWriteCheck::pending_injection(), + returning: None, + rls_filters: Vec::new(), + }) + } + + /// A timeseries ingest locks its collection `Intent` and nothing else. + /// It waits rather than routes when contended. + #[test] + fn a_timeseries_ingest_announces_its_collection_and_never_routes() { + let (keys, sequenced) = keys_of(&ts_ingest()); + assert!(!sequenced); + assert_eq!(keys.get(&coll("metrics")), Some(&LockMode::Intent)); + assert_eq!(keys.len(), 1); + } + + /// A columnar insert of every intent locks its rows `Exclusive` and its + /// collection `Intent`. It waits rather than routes when contended. + #[test] + fn a_columnar_insert_locks_its_rows_and_never_routes() { + for intent in [ + ColumnarInsertIntent::Insert, + ColumnarInsertIntent::InsertIfAbsent, + ColumnarInsertIntent::Put, + ColumnarInsertIntent::InsertUnique, + ] { + let (keys, sequenced) = keys_of(&columnar_insert(intent)); + assert!(!sequenced, "{intent:?}"); + assert_eq!( + keys.get(&coll("events")), + Some(&LockMode::Intent), + "{intent:?}" + ); + for surrogate in [4, 5] { + assert_eq!( + keys.get(&surrogate_key("events", surrogate)), + Some(&LockMode::Exclusive), + "{intent:?}" + ); + } + assert_eq!(keys.len(), 3, "{intent:?}"); + } + } + + /// Two concurrent ingests of one collection hold their keys together. A + /// truncate of the collection cannot take it while either holds. + #[test] + fn concurrent_ingests_admit_together_and_fence_a_truncate() { + let mut table = LockManager::new(); + let (first, _) = keys_of(&ts_ingest()); + let (second, _) = keys_of(&ts_ingest()); + assert!(table.try_acquire(TxnId::new(TxnId::AUTOCOMMIT_EPOCH, 0), first)); + assert!(table.try_acquire(TxnId::new(TxnId::AUTOCOMMIT_EPOCH, 1), second)); + let truncate = PhysicalPlan::Timeseries(TimeseriesOp::Truncate { + collection: qualified("metrics"), + restart_identity: false, + }); + let (truncate_keys, _) = keys_of(&truncate); + assert_eq!( + truncate_keys.get(&coll("metrics")), + Some(&LockMode::Exclusive) + ); + assert!(!table.try_acquire(TxnId::new(TxnId::AUTOCOMMIT_EPOCH, 2), truncate_keys)); + } + + /// A committed redo locks every collection it writes `Exclusive`. + #[test] + fn a_whole_collection_request_locks_each_collection_exclusive() { + let request = collection_admission_keys(&[ + qualified("a").as_str().to_owned(), + qualified("b").as_str().to_owned(), + ]); + assert!(!request.sequenced); + assert_eq!(request.keys.get(&coll("a")), Some(&LockMode::Exclusive)); + assert_eq!(request.keys.get(&coll("b")), Some(&LockMode::Exclusive)); + assert_eq!(request.keys.len(), 2); + } +} diff --git a/nodedb/src/control/server/shared/write_admission/gate.rs b/nodedb/src/control/server/shared/write_admission/gate.rs index 502a95c6c..af9456998 100644 --- a/nodedb/src/control/server/shared/write_admission/gate.rs +++ b/nodedb/src/control/server/shared/write_admission/gate.rs @@ -2,32 +2,45 @@ //! The write-admission gate. //! -//! Every write-class `PhysicalPlan` — regardless of transport or path — passes -//! through [`admit`] before it is enqueued to a Data-Plane core. The gate -//! decides one of four outcomes per write: +//! Every write-class `PhysicalPlan` passes through [`admit`] before it is +//! ordered: an autocommit write that applies on this node alone before its +//! enqueue, and a replicated write on its data-group leader before its +//! propose (see `leader_gate`). The gate decides one of five outcomes: //! -//! - [`WriteAdmission::FastPath`] — an uncontended POINT write whose exact -//! deterministic lock keys were acquired here. It carries a RAII -//! [`WriteAdmissionGuard`] the caller holds across the enqueue + response; the -//! guard releases the keys on drop. This is the normal autocommit path. +//! - [`WriteAdmission::FastPath`] — every lock key of the write was free and +//! is held now by the RAII [`WriteAdmissionGuard`]. A local write holds it +//! across its enqueue. A replicated write holds it until its leader starts +//! the entry's apply. The guard releases the keys on drop. //! - [`WriteAdmission::FastPathBlocking`] — a single-node POINT write for a //! vShard with no Calvin scheduler. There is no lock table to fence against, //! so the caller awaits a FIFO-fair per-key async order-lock before the WAL //! append + enqueue, serializing concurrent same-key writes in arrival order. -//! - [`WriteAdmission::RouteToCalvin`] — a point write whose keys are currently -//! held by a pending commit (acquire returned `Blocked`), OR any predicate / -//! bulk / multi-home write. The caller submits it through the deterministic -//! scheduler, which queues it FIFO behind the holder and applies it in order. +//! - [`WriteAdmission::RouteToCalvin`] — a key is held, and a Calvin +//! transaction can sequence the write, or the write homes on several +//! vShards. The caller submits it through the deterministic scheduler, which +//! queues it FIFO behind the holder and applies it in order. +//! - [`WriteAdmission::Wait`] — a key is held, and no Calvin transaction can +//! sequence the write. The caller awaits [`AdmissionWait::acquire`]. //! - [`WriteAdmission::ExemptRead`] — a non-write (read / meta op), or a //! Calvin-scheduled apply that already holds its locks. //! -//! The fence holds because the fast path and the scheduler share the SAME +//! Every write shape has a lock request (see `admission_keys`): its row keys +//! `Exclusive` plus its collection `Intent`, or its collection `Exclusive`. +//! +//! The fence holds because the gate and the scheduler share the SAME //! `Arc>` (via [`CalvinLocalState::lock_managers`]): a //! commit's lock validation calls `acquire` on the same key, is `Blocked`, and //! waits; whoever takes the OS mutex first wins, with no time-of-check / //! time-of-use gap. //! +//! The gate cannot deadlock with a Calvin transaction. It takes a write's keys +//! all at once with `try_acquire`, or none of them. A contended write routes +//! to the scheduler, which orders it by sequencer position, or waits holding +//! no key. A write that holds keys waits only for its own enqueue, which +//! never waits for a lock. +//! //! [`CalvinLocalState::lock_managers`]: crate::control::state::CalvinLocalState::lock_managers +//! [`AdmissionWait::acquire`]: super::wait::AdmissionWait::acquire use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; @@ -36,20 +49,18 @@ use tokio::sync::mpsc::UnboundedSender; use crate::bridge::envelope::PhysicalPlan; use crate::control::cluster::calvin::scheduler::driver::core::routing::{PlanRouting, plan_vshard}; -use crate::control::cluster::calvin::scheduler::lock_manager::{ - LockKey, LockManager, LockMode, TxnId, -}; -use crate::control::planner::calvin::is_dependent_predicate; +use crate::control::cluster::calvin::scheduler::lock_manager::{LockKey, LockManager, TxnId}; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId, VShardId}; -use nodedb_physical::physical_plan::MetaOp; -use super::lock_keys::{plan_lock_keys, plan_row_key}; +use super::admission_keys::{AdmissionKeys, is_calvin_apply, plan_admission_keys}; +use super::lock_keys::plan_row_key; use super::predicate::plan_is_write; +use super::wait::AdmissionWait; use super::write_order_lock::KeyedWriteOrderLock; /// Count of writes the gate routed to the deterministic scheduler instead of -/// the fast path (either a `Blocked` point write or a non-point write). Read by +/// the fast path (a contended write, or a write on several vShards). Read by /// the fence tests. static ROUTED_TO_CALVIN: AtomicU64 = AtomicU64::new(0); @@ -73,11 +84,11 @@ pub struct WriteTarget<'a> { /// The gate's decision for one write. See the module docs. pub enum WriteAdmission { - /// Uncontended point write admitted to the fast path. `guard` is `Some` when - /// real keys were acquired (a scheduler is active for the vShard) and `None` - /// when no lock manager is registered and the write carries no single point - /// key to serialize on (predicate / bulk / uncovered shapes — left unordered - /// for now). Either way the caller holds it across enqueue + response. + /// Uncontended write admitted to the fast path. `guard` is `Some` when + /// its keys were acquired. It is `None` when no Calvin scheduler runs for + /// the vShard and the write has no point key, or when the write names no + /// lock key. No Calvin transaction can then conflict with it, and the + /// vShard's write-order fence orders it against other writes. FastPath { guard: Option }, /// POINT write on a vShard with no Calvin scheduler registered. There is no /// lock table to fence against, but concurrent same-key writes must still @@ -93,6 +104,9 @@ pub enum WriteAdmission { }, /// Submit the write through the deterministic Calvin scheduler. RouteToCalvin, + /// A key is held, and no Calvin transaction can sequence the write. The + /// caller awaits [`AdmissionWait::acquire`] before it orders the write. + Wait(AdmissionWait), /// A non-write, or an already-locked Calvin apply — no fence needed. ExemptRead, } @@ -125,6 +139,21 @@ pub struct WriteAdmissionGuard { promotion_sender: Option>>, } +impl WriteAdmissionGuard { + /// The guard of the keys `txn` holds on `lock_manager`. + pub(super) fn new( + lock_manager: Arc>, + txn: TxnId, + promotion_sender: Option>>, + ) -> Self { + Self { + lock_manager, + txn, + promotion_sender, + } + } +} + impl Drop for WriteAdmissionGuard { fn drop(&mut self) { // Ordering is load-bearing: take the lock-manager mutex, release the @@ -154,94 +183,72 @@ impl Drop for WriteAdmissionGuard { } } -/// Admit a plan destined for a Data-Plane core. +/// Admit a write-class plan. /// /// Synchronous: never awaits and never parks — it only *chooses* the outcome; -/// any per-key await happens in the caller. `RouteToCalvin` is returned ONLY -/// when a deterministic scheduler is actually registered for the write's vShard. -/// With no scheduler (single-node / no-Calvin — the common case) a point write -/// returns `FastPathBlocking` carrying the global keyed order-lock so concurrent -/// same-key writes still serialize; every other shape fast-paths unfenced. -pub fn admit(shared: &SharedState, target: &WriteTarget<'_>) -> WriteAdmission { - // A Calvin-scheduled apply already holds its locks (acquired by the - // scheduler); it must never re-acquire at the gate. Defensive — these ops - // do not normally reach the gate. - if matches!( - target.plan, - PhysicalPlan::Meta( - MetaOp::CalvinExecuteStatic { .. } - | MetaOp::CalvinExecuteActive { .. } - | MetaOp::CalvinFlush { .. } - | MetaOp::CalvinDrop { .. } - | MetaOp::CalvinResolve { .. } - ) - ) { - return WriteAdmission::ExemptRead; - } +/// any await happens in the caller. `RouteToCalvin` and `Wait` are returned +/// only when a deterministic scheduler is registered for the write's vShard. +/// With no scheduler (single-node / no-Calvin) a point write returns +/// `FastPathBlocking` carrying the global keyed order-lock so concurrent +/// same-key writes still serialize, and every other shape takes the fast path +/// with no key: no Calvin transaction runs to conflict with it. +pub fn admit(shared: &SharedState, target: &WriteTarget<'_>) -> crate::Result { + admit_routed(shared, target, true) +} - if !plan_is_write(target.plan) { - return WriteAdmission::ExemptRead; +/// [`admit`], where `may_route` says whether a contended write may run +/// through the scheduler (see `calvin_route_keeps`). A contended write that +/// may not waits for its keys instead. +pub(crate) fn admit_routed( + shared: &SharedState, + target: &WriteTarget<'_>, + may_route: bool, +) -> crate::Result { + // A Calvin-scheduled apply already holds its locks (acquired by the + // scheduler); it must never re-acquire at the gate. + if is_calvin_apply(target.plan) || !plan_is_write(target.plan) { + return Ok(WriteAdmission::ExemptRead); } - // Only two write shapes participate in the fence: a single-home POINT write - // (Document / KV / Vector / single-home graph edge — a statically-known - // deterministic key) and a single-shard PREDICATE write (BulkUpdate / - // BulkDelete — its write set discovered by scheduler reconnaissance). Every - // other write — batch, INSERT..SELECT, upsert, CRDT, columnar / timeseries / - // spatial / array, and cross-home edges — has no Calvin lock representation - // and fast-paths unchanged. - let point_keys = plan_lock_keys(target.plan); - let is_predicate = is_dependent_predicate(target.plan); - let vshard = match &point_keys { - Some((v, _)) => *v, - None if is_predicate => match plan_vshard(target.plan) { - PlanRouting::Vshards(v) => match v.as_slice() { - [v] => *v, - _ => return WriteAdmission::FastPath { guard: None }, - }, - PlanRouting::ControlPlaneOnly | PlanRouting::NotAWrite | PlanRouting::Unroutable(_) => { - return WriteAdmission::FastPath { guard: None }; - } + let vshard = match plan_vshard(target.plan) { + PlanRouting::Vshards(homes) => match homes.as_slice() { + [home] => *home, + // Only the scheduler takes keys on several lock tables as one + // request, in sequencer order. + homes => return Ok(admit_multi_home(shared, homes)), }, - None => return WriteAdmission::FastPath { guard: None }, + // A write whose plan names no home orders on the vShard it targets. + PlanRouting::ControlPlaneOnly | PlanRouting::NotAWrite | PlanRouting::Unroutable(_) => { + target.vshard_id + } }; - // Availability gate: with no scheduler registered for this vShard there is no - // Calvin lock table to fence against. A POINT write still needs per-key - // arrival-order serialization so WAL-LSN order equals Data-Plane apply order - // per key — hand it the global keyed order-lock, on which concurrent same-key - // writers queue FIFO while distinct keys never contend. A predicate write has - // no single static point key here, so it stays unordered on the fast path - // (widening that coverage is a later unit). - let Some(lock_manager) = shared - .calvin - .lock_managers - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get(&vshard.as_u32()) - .map(Arc::clone) - else { - return match point_keys.and_then(|_| plan_row_key(target.plan)) { + let Some(lock_manager) = lock_manager_of(shared, vshard) else { + return Ok(match plan_row_key(target.plan) { Some(key) => WriteAdmission::FastPathBlocking { key, keyed_lock: Arc::clone(&shared.write_order_locks), }, None => WriteAdmission::FastPath { guard: None }, - }; + }); }; - // Calvin IS running for this vShard. A predicate write has no static point - // key to acquire; the scheduler discovers its write set, so route it. - let Some((_v, keys)) = point_keys else { - ROUTED_TO_CALVIN.fetch_add(1, Ordering::Relaxed); - return WriteAdmission::RouteToCalvin; - }; + let mut request = + plan_admission_keys(shared, target.tenant_id, target.database_id, target.plan)?; + request.sequenced &= may_route; + Ok(admit_request(shared, vshard, lock_manager, request)) +} - // Point write: mint a holder id in the reserved band so it never collides - // with a real Calvin schedule position, then probe the exact keys WITHOUT - // blocking. `try_acquire` never enqueues a waiter on the contended path, so a - // routed write leaves no orphaned autocommit holder that a later `release` - // can promote to an unowned (never-released) lock. +/// Admit the lock request `request` on `vshard`, whose lock table is +/// `lock_manager`. +pub(crate) fn admit_request( + shared: &SharedState, + vshard: VShardId, + lock_manager: Arc>, + request: AdmissionKeys, +) -> WriteAdmission { + // A holder id in the reserved band never collides with a real Calvin + // schedule position. let txn = TxnId::new( TxnId::AUTOCOMMIT_EPOCH, shared @@ -249,39 +256,227 @@ pub fn admit(shared: &SharedState, target: &WriteTarget<'_>) -> WriteAdmission { .autocommit_lock_seq .fetch_add(1, Ordering::Relaxed), ); - let acquired = { - let mut lm = lock_manager.lock().unwrap_or_else(|p| p.into_inner()); - lm.try_acquire( - txn, - keys.into_iter() - .map(|key| (key, LockMode::Exclusive)) - .collect(), - ) - }; - if acquired { - // Look up this vShard's promotion channel so the guard can hand any - // scheduler waiter it promotes on drop back to the scheduler for - // dispatch. `None` only if no scheduler is registered — but a registered - // lock manager without a promotion sender must not happen, since both - // are inserted together per vShard. - let promotion_sender = shared - .calvin - .promotion_senders - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get(&vshard.as_u32()) - .cloned(); - WriteAdmission::FastPath { - guard: Some(WriteAdmissionGuard { + // Every guard hands the scheduler waiters it promotes on drop back to + // the scheduler for dispatch. A lock manager and its promotion sender + // are registered together per vShard. + let promotion_sender = shared + .calvin + .promotion_senders + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&vshard.as_u32()) + .cloned(); + let admission = probe(lock_manager, txn, request, promotion_sender); + if matches!(admission, WriteAdmission::RouteToCalvin) { + ROUTED_TO_CALVIN.fetch_add(1, Ordering::Relaxed); + } + admission +} + +/// Probe `lock_manager` for every key of `request` at once, WITHOUT +/// blocking. `try_acquire` never enqueues a waiter on the contended path, so +/// a routed or waiting write leaves no orphaned holder that a later `release` +/// can promote to an unowned lock. +fn probe( + lock_manager: Arc>, + txn: TxnId, + request: AdmissionKeys, + promotion_sender: Option>>, +) -> WriteAdmission { + let AdmissionKeys { keys, sequenced } = request; + if keys.is_empty() { + return WriteAdmission::FastPath { guard: None }; + } + // A write that waits probes again with the same request. + let retry_keys = (!sequenced).then(|| keys.clone()); + let acquired = lock_manager + .lock() + .unwrap_or_else(|p| p.into_inner()) + .try_acquire(txn, keys); + match (acquired, retry_keys) { + (true, _) => WriteAdmission::FastPath { + guard: Some(WriteAdmissionGuard::new( lock_manager, txn, promotion_sender, - }), - } - } else { - // A pending commit (or another fast-path write) holds a key: route behind - // it via the scheduler. Nothing was acquired or enqueued here. + )), + }, + // A holder or an earlier waiter has a key: the scheduler queues the + // write FIFO behind it. + (false, None) => WriteAdmission::RouteToCalvin, + (false, Some(keys)) => WriteAdmission::Wait(AdmissionWait::new( + lock_manager, + txn, + keys, + promotion_sender, + )), + } +} + +/// Admit a write homed on several vShards: the scheduler sequences it when +/// any home runs one. With no scheduler on any home, no Calvin transaction +/// conflicts with it. +fn admit_multi_home(shared: &SharedState, homes: &[VShardId]) -> WriteAdmission { + if homes + .iter() + .any(|home| lock_manager_of(shared, *home).is_some()) + { ROUTED_TO_CALVIN.fetch_add(1, Ordering::Relaxed); WriteAdmission::RouteToCalvin + } else { + WriteAdmission::FastPath { guard: None } + } +} + +/// The lock table of `vshard`'s Calvin scheduler on this node, if one runs. +pub(crate) fn lock_manager_of( + shared: &SharedState, + vshard: VShardId, +) -> Option>> { + shared + .calvin + .lock_managers + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&vshard.as_u32()) + .map(Arc::clone) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + use crate::control::cluster::calvin::scheduler::lock_manager::{AcquireOutcome, LockMode}; + + fn coll() -> LockKey { + LockKey::Collection { + collection: Arc::from("c"), + } + } + + fn row(surrogate: u32) -> LockKey { + LockKey::Surrogate { + collection: Arc::from("c"), + surrogate, + } + } + + fn row_write(surrogate: u32) -> BTreeMap { + BTreeMap::from([ + (row(surrogate), LockMode::Exclusive), + (coll(), LockMode::Intent), + ]) + } + + fn request(keys: BTreeMap, sequenced: bool) -> AdmissionKeys { + AdmissionKeys { keys, sequenced } + } + + fn autocommit(position: u32) -> TxnId { + TxnId::new(TxnId::AUTOCOMMIT_EPOCH, position) + } + + /// Two row writes of distinct rows hold the collection `Intent` together. + /// The guard passes each key in its own mode. + #[test] + fn row_writes_of_distinct_rows_admit_together() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let first = probe( + Arc::clone(&lock_manager), + autocommit(0), + request(row_write(1), true), + None, + ); + let second = probe( + Arc::clone(&lock_manager), + autocommit(1), + request(row_write(2), true), + None, + ); + assert!(matches!(first, WriteAdmission::FastPath { guard: Some(_) })); + assert!(matches!( + second, + WriteAdmission::FastPath { guard: Some(_) } + )); + drop((first, second)); + assert_eq!(lock_manager.lock().expect("table").lock_count(), 0); + } + + /// A Calvin truncate holds the collection `Exclusive`. A sequenced row + /// write routes to the scheduler, and an unsequenced one waits. + #[test] + fn a_contended_write_routes_when_sequenced_and_waits_otherwise() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let truncate = TxnId::new(3, 0); + assert_eq!( + lock_manager + .lock() + .expect("table") + .acquire(truncate, BTreeMap::from([(coll(), LockMode::Exclusive)])), + AcquireOutcome::Ready + ); + let routed = probe( + Arc::clone(&lock_manager), + autocommit(0), + request(row_write(1), true), + None, + ); + assert!(matches!(routed, WriteAdmission::RouteToCalvin)); + let waits = probe( + Arc::clone(&lock_manager), + autocommit(1), + request(row_write(1), false), + None, + ); + assert!(matches!(waits, WriteAdmission::Wait(_))); + assert_eq!( + lock_manager.lock().expect("table").holder_count(), + 1, + "a refused probe takes no key" + ); + } + + /// A Calvin transaction that needs a key a fast-path write holds waits + /// for the guard's drop, which promotes it. + #[test] + fn a_calvin_writer_waits_for_the_fast_path_guard() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let admitted = probe( + Arc::clone(&lock_manager), + autocommit(0), + request(row_write(1), true), + None, + ); + assert!(matches!( + admitted, + WriteAdmission::FastPath { guard: Some(_) } + )); + let truncate = TxnId::new(3, 0); + assert_eq!( + lock_manager + .lock() + .expect("table") + .acquire(truncate, BTreeMap::from([(coll(), LockMode::Exclusive)])), + AcquireOutcome::Blocked + ); + drop(admitted); + assert!( + lock_manager + .lock() + .expect("table") + .is_ready(truncate, &BTreeMap::from([(coll(), LockMode::Exclusive)])) + ); + } + + /// A write that names no lock key takes the fast path with no guard. + #[test] + fn a_write_with_no_keys_takes_no_guard() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let admission = probe(lock_manager, autocommit(0), AdmissionKeys::default(), None); + assert!(matches!( + admission, + WriteAdmission::FastPath { guard: None } + )); } } diff --git a/nodedb/src/control/server/shared/write_admission/holds.rs b/nodedb/src/control/server/shared/write_admission/holds.rs new file mode 100644 index 000000000..e1e108a13 --- /dev/null +++ b/nodedb/src/control/server/shared/write_admission/holds.rs @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The lock keys replicated writes hold on their data-group leader. +//! +//! The leader's write gate takes a write's keys before its propose. The keys +//! stay held until the leader's apply loop starts the entry: its enqueue +//! returned, or it concluded without one. The apply loop enqueues a group's +//! writes on their cores in log order, and a core runs its queue in order. A +//! Calvin transaction granted a key after the release therefore stages on +//! the core after the write, and reads the write's effects. +//! +//! The keys are never held across a Data-Plane apply. A write that holds keys +//! waits only for the apply loop to start its entry, which never waits for a +//! lock. So no held key waits for a Calvin transaction to finish. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::Mutex; + +use crate::types::VShardId; + +use super::gate::WriteAdmissionGuard; + +/// One write's keys, held for their `Drop`. +struct Held { + vshard: VShardId, + _guard: WriteAdmissionGuard, +} + +#[derive(Default)] +struct Inner { + /// Held keys by the `(group_id, log_index)` of the write's entry. + holds: BTreeMap<(u64, u64), Vec>, + /// Per group, the highest log index whose apply started on this node. + started: HashMap, +} + +/// Every replicated write's held keys on this node, by entry. +#[derive(Default)] +pub struct AdmissionHolds { + inner: Mutex, +} + +impl AdmissionHolds { + pub fn new() -> Self { + Self::default() + } + + /// Hold `guard` until the apply of entry `log_index` of `group_id` + /// starts on this node. An entry whose apply started already releases + /// it at once. + pub fn hold( + &self, + group_id: u64, + log_index: u64, + vshard: VShardId, + guard: WriteAdmissionGuard, + ) { + let held = Held { + vshard, + _guard: guard, + }; + // Dropped after the table's mutex: a guard's drop takes the lock + // table's mutex. + let _released = { + let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); + if inner + .started + .get(&group_id) + .is_some_and(|started| *started >= log_index) + { + Some(held) + } else { + inner + .holds + .entry((group_id, log_index)) + .or_default() + .push(held); + None + } + }; + } + + /// The apply of entry `log_index` of `group_id` started on this node, + /// after every entry of the group below it. Release the keys every write + /// through it holds. + pub fn release_through(&self, group_id: u64, log_index: u64) { + let _released: Vec> = { + let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); + let started = inner.started.entry(group_id).or_default(); + *started = (*started).max(log_index); + let through: Vec<(u64, u64)> = inner + .holds + .range((group_id, 0)..=(group_id, log_index)) + .map(|(key, _)| *key) + .collect(); + through + .into_iter() + .filter_map(|key| inner.holds.remove(&key)) + .collect() + }; + } + + /// The scheduler of `vshard` stopped on this node, and its lock table + /// with it. Drop every hold on that table. + pub fn release_vshard(&self, vshard: VShardId) { + let _released: Vec = { + let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner()); + let mut released = Vec::new(); + inner.holds.retain(|_, held| { + let (gone, kept): (Vec, Vec) = std::mem::take(held) + .into_iter() + .partition(|h| h.vshard == vshard); + released.extend(gone); + *held = kept; + !held.is_empty() + }); + released + }; + } + + /// The number of entries whose writes hold keys. + #[cfg(test)] + fn held_entries(&self) -> usize { + self.inner + .lock() + .unwrap_or_else(|p| p.into_inner()) + .holds + .len() + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::Arc; + + use super::*; + use crate::control::cluster::calvin::scheduler::lock_manager::{ + LockKey, LockManager, LockMode, TxnId, + }; + + fn guard(lock_manager: &Arc>, position: u32) -> WriteAdmissionGuard { + let txn = TxnId::new(TxnId::AUTOCOMMIT_EPOCH, position); + let key = LockKey::Surrogate { + collection: Arc::from("c"), + surrogate: position, + }; + assert!( + lock_manager + .lock() + .expect("table") + .try_acquire(txn, BTreeMap::from([(key, LockMode::Exclusive)])) + ); + WriteAdmissionGuard::new(Arc::clone(lock_manager), txn, None) + } + + fn held_keys(lock_manager: &Arc>) -> usize { + lock_manager.lock().expect("table").lock_count() + } + + /// A write's keys stay held until its entry starts, and no longer. + #[test] + fn keys_release_when_their_entry_starts() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let holds = AdmissionHolds::new(); + holds.hold(4, 10, VShardId::new(1), guard(&lock_manager, 1)); + holds.hold(4, 11, VShardId::new(1), guard(&lock_manager, 2)); + assert_eq!(held_keys(&lock_manager), 2); + holds.release_through(4, 10); + assert_eq!(held_keys(&lock_manager), 1); + assert_eq!(holds.held_entries(), 1); + holds.release_through(4, 11); + assert_eq!(held_keys(&lock_manager), 0); + } + + /// An entry that started before its proposer registered the hold + /// releases the keys at once. + #[test] + fn a_hold_on_a_started_entry_releases_at_once() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let holds = AdmissionHolds::new(); + holds.release_through(4, 12); + holds.hold(4, 12, VShardId::new(1), guard(&lock_manager, 1)); + assert_eq!(held_keys(&lock_manager), 0); + assert_eq!(holds.held_entries(), 0); + } + + /// Another group's start releases nothing. + #[test] + fn groups_release_apart() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let holds = AdmissionHolds::new(); + holds.hold(4, 10, VShardId::new(1), guard(&lock_manager, 1)); + holds.release_through(5, 20); + assert_eq!(held_keys(&lock_manager), 1); + } + + /// A stopped scheduler drops the holds on its table. + #[test] + fn a_stopped_vshard_drops_its_holds() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let holds = AdmissionHolds::new(); + holds.hold(4, 10, VShardId::new(1), guard(&lock_manager, 1)); + holds.hold(4, 10, VShardId::new(2), guard(&lock_manager, 2)); + holds.release_vshard(VShardId::new(1)); + assert_eq!(held_keys(&lock_manager), 1); + assert_eq!(holds.held_entries(), 1); + } +} diff --git a/nodedb/src/control/server/shared/write_admission/leader_gate.rs b/nodedb/src/control/server/shared/write_admission/leader_gate.rs new file mode 100644 index 000000000..b3eccc32f --- /dev/null +++ b/nodedb/src/control/server/shared/write_admission/leader_gate.rs @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The write gate a data-group leader runs on every proposal to its group. +//! +//! A replicated write applies on every replica in log order, with its +//! ordering decided once: on the leader, before the entry enters the log. +//! [`LeaderWriteGate`] admits the entry there with the same gate a local +//! write passes: +//! +//! - Uncontended: the leader proposes the entry, and its keys stay held until +//! the leader's apply loop starts the entry (see `holds`). +//! - Contended, and the write routes (see `admission_keys`): the propose +//! refuses with `RouteToSequencer`, and the proposer submits the write to +//! the Calvin sequencer. An append never routes. +//! - Contended otherwise: the gate waits holding no key, until the keys are +//! free or the proposer's deadline passes. A forwarded entry carries what +//! remains of that deadline. +//! +//! Followers apply the committed entry with no gate: the leader ordered it. + +use std::sync::{Arc, Weak}; + +use nodedb_cluster::{CalvinError, ClusterError, DataProposeGate, ProposeHold}; + +use crate::control::state::SharedState; +use crate::control::wal_replication::decode::decode_parsed_entry; +use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite}; +use crate::types::VShardId; +use nodedb_types::QualifiedCollection; + +use super::admission_keys::collection_admission_keys; +use super::gate::{ + WriteAdmission, WriteAdmissionGuard, WriteTarget, admit_request, admit_routed, lock_manager_of, +}; +use super::route::calvin_route_keeps; + +/// The host side of [`DataProposeGate`]. Weak: `SharedState` outlives the +/// Raft loop that holds the gate only while the node runs. +pub struct LeaderWriteGate { + state: Weak, +} + +impl LeaderWriteGate { + pub fn new(state: &Arc) -> Self { + Self { + state: Arc::downgrade(state), + } + } +} + +#[async_trait::async_trait] +impl DataProposeGate for LeaderWriteGate { + async fn admit( + &self, + vshard_id: u32, + entry: &[u8], + deadline: tokio::time::Instant, + ) -> nodedb_cluster::Result>> { + // A node that is shutting down proposes nothing its gate must order. + let Some(state) = self.state.upgrade() else { + return Ok(None); + }; + let vshard = VShardId::new(vshard_id); + let guard = match admit_entry(&state, vshard, entry).map_err(gate_error)? { + WriteAdmission::ExemptRead + | WriteAdmission::FastPath { guard: None } + // No scheduler runs for the vShard here: the log alone orders + // the write. + | WriteAdmission::FastPathBlocking { .. } => return Ok(None), + WriteAdmission::FastPath { guard: Some(guard) } => guard, + WriteAdmission::RouteToCalvin => { + return Err(ClusterError::Calvin(CalvinError::RouteToSequencer)); + } + WriteAdmission::Wait(wait) => wait.acquire(deadline).await.map_err(gate_error)?, + }; + Ok(Some(Box::new(LandedHold { + state, + vshard, + guard, + }))) + } +} + +/// An admitted entry's keys, held until the entry lands in the log. +struct LandedHold { + state: Arc, + vshard: VShardId, + guard: WriteAdmissionGuard, +} + +impl ProposeHold for LandedHold { + fn landed(self: Box, group_id: u64, log_index: u64) { + let LandedHold { + state, + vshard, + guard, + } = *self; + state + .calvin + .admission_holds + .hold(group_id, log_index, vshard, guard); + } +} + +/// What the gate locks for one replicated entry. +enum EntryScope { + /// The entry writes no row, or its Calvin scheduler holds its locks. + Exempt, + /// The entry writes these collections whole. + Collections(Vec), + /// The entry carries a plan the gate locks as it locks a local write. + Plan, +} + +/// Admit the encoded entry `entry`, proposed to `vshard`'s group. +pub(crate) fn admit_entry( + state: &SharedState, + vshard: VShardId, + entry: &[u8], +) -> crate::Result { + // Bytes that are not a replicated entry apply as nothing on any replica. + let Some(entry) = ReplicatedEntry::from_bytes(entry) else { + return Ok(WriteAdmission::ExemptRead); + }; + let database_id = crate::types::DatabaseId::new(entry.database_id); + match entry_scope(&entry.write, database_id) { + EntryScope::Exempt => Ok(WriteAdmission::ExemptRead), + EntryScope::Collections(collections) => Ok(admit_collections(state, vshard, &collections)), + EntryScope::Plan => { + let Some((_, (tenant_id, _, plan, _))) = decode_parsed_entry(&entry)? else { + return Ok(WriteAdmission::ExemptRead); + }; + // An entry the scheduler cannot apply as proposed waits for its + // keys rather than route. + let may_route = calvin_route_keeps( + &plan, + crate::event::EventSource::from(entry.event_source), + entry.restore_id, + ); + admit_routed( + state, + &WriteTarget { + tenant_id, + database_id, + vshard_id: vshard, + plan: &plan, + }, + may_route, + ) + } + } +} + +/// The lock scope of `write`, an entry of `database_id`. +fn entry_scope(write: &ReplicatedWrite, database_id: crate::types::DatabaseId) -> EntryScope { + match write { + // Calvin bookkeeping, a backup cut, a topic message, a surrogate + // binding and an array schema write no row. + ReplicatedWrite::CalvinReadResult { .. } + | ReplicatedWrite::CutBarrier { .. } + | ReplicatedWrite::TopicPublish { .. } + | ReplicatedWrite::SurrogateBind { .. } + | ReplicatedWrite::ArraySchema { .. } => EntryScope::Exempt, + // A Calvin transaction's redo carries its sequencer stamp, and its + // scheduler holds its locks. A session transaction's redo writes + // every collection it names. + ReplicatedWrite::TransactionRedo { + redo, collections, .. + } => match redo.calvin_stamp { + Some(_) => EntryScope::Exempt, + None => EntryScope::Collections(collections.clone()), + }, + ReplicatedWrite::ArrayOp { array, .. } => EntryScope::Collections(vec![ + QualifiedCollection::new(database_id, array) + .as_str() + .to_owned(), + ]), + ReplicatedWrite::PointPut { .. } + | ReplicatedWrite::PointInsert { .. } + | ReplicatedWrite::PointDelete { .. } + | ReplicatedWrite::PointUpdate { .. } + | ReplicatedWrite::DocUpsert { .. } + | ReplicatedWrite::DocBatchInsert { .. } + | ReplicatedWrite::VectorInsert { .. } + | ReplicatedWrite::VectorBatchInsert { .. } + | ReplicatedWrite::VectorDelete { .. } + | ReplicatedWrite::SetVectorParams { .. } + | ReplicatedWrite::SparseInsert { .. } + | ReplicatedWrite::SparseDelete { .. } + | ReplicatedWrite::MultiVectorInsert { .. } + | ReplicatedWrite::MultiVectorDelete { .. } + | ReplicatedWrite::DeleteBySurrogate { .. } + | ReplicatedWrite::DirectUpsert { .. } + | ReplicatedWrite::CrdtApply { .. } + | ReplicatedWrite::ColumnarIngest { .. } + | ReplicatedWrite::TimeseriesIngest { .. } + | ReplicatedWrite::FtsIndex { .. } + | ReplicatedWrite::FtsDelete { .. } + | ReplicatedWrite::SpatialInsert { .. } + | ReplicatedWrite::SpatialDelete { .. } + | ReplicatedWrite::EdgePut { .. } + | ReplicatedWrite::EdgeDelete { .. } + | ReplicatedWrite::SetNodeLabels { .. } + | ReplicatedWrite::RemoveNodeLabels { .. } + | ReplicatedWrite::EdgePutBatch { .. } + | ReplicatedWrite::EdgeDeleteBatch { .. } + | ReplicatedWrite::KvPut { .. } + | ReplicatedWrite::KvDelete { .. } + | ReplicatedWrite::KvInsert { .. } + | ReplicatedWrite::KvInsertIfAbsent { .. } + | ReplicatedWrite::KvInsertOnConflictUpdate { .. } + | ReplicatedWrite::KvBatchPut { .. } + | ReplicatedWrite::KvExpire { .. } + | ReplicatedWrite::KvPersist { .. } + | ReplicatedWrite::KvIncr { .. } + | ReplicatedWrite::KvIncrFloat { .. } + | ReplicatedWrite::KvCas { .. } + | ReplicatedWrite::KvGetSet { .. } + | ReplicatedWrite::KvRegisterSortedIndex { .. } + | ReplicatedWrite::KvDropSortedIndex { .. } + | ReplicatedWrite::KvRegisterIndex { .. } + | ReplicatedWrite::KvDropIndex { .. } + | ReplicatedWrite::KvFieldSet { .. } + | ReplicatedWrite::KvTransfer { .. } + | ReplicatedWrite::KvTransferItem { .. } + | ReplicatedWrite::BulkDml { .. } + | ReplicatedWrite::ColumnarBulkDml { .. } + | ReplicatedWrite::InsertSelect { .. } + | ReplicatedWrite::CrdtImportCollection { .. } + | ReplicatedWrite::CrdtListInsert { .. } + | ReplicatedWrite::CrdtListDelete { .. } + | ReplicatedWrite::CrdtListMove { .. } + | ReplicatedWrite::CrdtDocUpsert { .. } + | ReplicatedWrite::CrdtDocDelete { .. } + | ReplicatedWrite::DocTruncate { .. } + | ReplicatedWrite::KvTruncate { .. } + | ReplicatedWrite::ConstraintChange { .. } + | ReplicatedWrite::ArrayCellPut { .. } + | ReplicatedWrite::ArrayCellDelete { .. } + | ReplicatedWrite::CrdtApplyFenced { .. } + | ReplicatedWrite::CrdtApplyAuthenticated { .. } + | ReplicatedWrite::DropVectorIndex { .. } + | ReplicatedWrite::ApplyBalanceDelta { .. } + | ReplicatedWrite::ColumnarBulkDmlResolved { .. } + | ReplicatedWrite::ColumnarTruncate { .. } + | ReplicatedWrite::TimeseriesTruncate { .. } + | ReplicatedWrite::KvResolvedWrite { .. } + | ReplicatedWrite::KvPredicateUpdate { .. } + | ReplicatedWrite::KvPredicateDelete { .. } + | ReplicatedWrite::DocumentResolvedWrite { .. } + | ReplicatedWrite::VectorDirectDelete { .. } + | ReplicatedWrite::VectorDirectTruncate { .. } + | ReplicatedWrite::VectorDirectUpdate { .. } + | ReplicatedWrite::VectorResolvedDirectWrite { .. } + | ReplicatedWrite::MergeApply { .. } + | ReplicatedWrite::UpdateFromJoinApply { .. } => EntryScope::Plan, + } +} + +/// Admit a write that locks each of `collections` whole. +fn admit_collections( + state: &SharedState, + vshard: VShardId, + collections: &[String], +) -> WriteAdmission { + match lock_manager_of(state, vshard) { + Some(lock_manager) => admit_request( + state, + vshard, + lock_manager, + collection_admission_keys(collections), + ), + None => WriteAdmission::FastPath { guard: None }, + } +} + +/// The cluster error a gate failure answers the proposer with. A deadline +/// keeps its class across the hop. Every other error crosses typed. +fn gate_error(error: crate::Error) -> ClusterError { + match error { + crate::Error::DeadlineExceeded { .. } => { + ClusterError::Calvin(CalvinError::AdmissionTimedOut) + } + other => { + let detail = format!("data-group write gate: {other}"); + ClusterError::ShardExecution { + error: Box::new( + crate::control::cluster::execution_error_wire::execution_error_to_typed(other), + ), + detail, + } + } + } +} diff --git a/nodedb/src/control/server/shared/write_admission/lock_keys.rs b/nodedb/src/control/server/shared/write_admission/lock_keys.rs index f0248182d..76af8e0a7 100644 --- a/nodedb/src/control/server/shared/write_admission/lock_keys.rs +++ b/nodedb/src/control/server/shared/write_admission/lock_keys.rs @@ -1,10 +1,12 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Point-write lock-key extraction for the write-admission fast path. +//! Point-write key extraction for the per-vShard write order. //! -//! [`plan_lock_keys`] maps a plan to the deterministic lock keys the fast path -//! must hold. Returns `Some` only for single-vShard, single-identity point -//! writes; predicate/bulk/multi-home writes route to Calvin instead. +//! [`plan_lock_keys`] maps a single-vShard, single-identity point write to +//! the keys that name its row. The write-order fence reads them to decide +//! whether admission's guard already orders the row. [`plan_row_key`] names +//! the key the keyed order lock serializes on when no Calvin scheduler runs. +//! The gate's own lock request comes from `admission_keys`. use std::collections::BTreeSet; use std::sync::Arc; @@ -15,9 +17,8 @@ use crate::control::cluster::calvin::scheduler::lock_manager::LockKey; use crate::types::VShardId; use nodedb_physical::physical_plan::{DocumentOp, GraphOp, KvOp, VectorOp, VectorWriteTargets}; -/// The vShard and exact lock-key set a POINT write must hold on the fast path. -/// Returns `None` (routes to Calvin) for any plan that isn't a single-home, -/// single-identity point write. +/// The vShard and the row keys of a POINT write. `None` for any plan that +/// isn't a single-home, single-identity point write. pub(crate) fn plan_lock_keys(plan: &PhysicalPlan) -> Option<(VShardId, BTreeSet)> { // `plan_vshard` returns two vShards for a cross-home graph edge, which has no // single `(vShard, keys)` representation and is ineligible for the fast path. diff --git a/nodedb/src/control/server/shared/write_admission/mod.rs b/nodedb/src/control/server/shared/write_admission/mod.rs index 6d292ca23..fb5e24e62 100644 --- a/nodedb/src/control/server/shared/write_admission/mod.rs +++ b/nodedb/src/control/server/shared/write_admission/mod.rs @@ -1,21 +1,29 @@ // SPDX-License-Identifier: BUSL-1.1 //! Control-Plane write-admission gate: the single seam every write-class -//! `PhysicalPlan` passes through before it is enqueued to a Data-Plane core. -//! Uncontended point writes take the fast path holding per-vShard deterministic -//! locks; contended and bulk writes route to the Calvin scheduler. +//! `PhysicalPlan` passes through before it is ordered. A local write passes +//! it before its enqueue, a replicated write on its data-group leader before +//! its propose. An uncontended write holds its per-vShard deterministic locks; +//! a contended write routes to the Calvin scheduler or waits. +pub mod admission_keys; pub mod gate; +pub mod holds; +pub mod leader_gate; pub mod lock_keys; pub mod predicate; pub mod route; +pub mod wait; pub mod write_order_fence; pub mod write_order_lock; pub use gate::{WriteAdmission, WriteAdmissionGuard, WriteTarget, admit, cp_routed_to_calvin}; +pub use holds::AdmissionHolds; +pub use leader_gate::LeaderWriteGate; pub use predicate::{ all_writes_bufferable, plan_is_write, plan_requires_txn_buffering, plan_writes_user_data, }; -pub use route::{bare_ok_response, route_write_to_calvin}; +pub use route::{bare_ok_response, calvin_route_keeps, route_write_to_calvin}; +pub use wait::AdmissionWait; pub(crate) use write_order_fence::order_row_write; pub use write_order_fence::{WriteOrder, WriteOrderFence}; pub use write_order_lock::KeyedWriteOrderLock; diff --git a/nodedb/src/control/server/shared/write_admission/route.rs b/nodedb/src/control/server/shared/write_admission/route.rs index b3d9e2d9f..ae5892b0c 100644 --- a/nodedb/src/control/server/shared/write_admission/route.rs +++ b/nodedb/src/control/server/shared/write_admission/route.rs @@ -3,17 +3,18 @@ //! Submit a gate-rejected write through the deterministic Calvin scheduler. //! //! When [`admit`](super::admit) returns [`WriteAdmission::RouteToCalvin`](super::WriteAdmission), -//! the caller hands the single write here. Two shapes reach this path, each -//! routed through the existing Calvin entry point that builds a VALID write set -//! for it: +//! the caller hands the single write here. A data-group leader's gate routes a +//! replicated write here too, through its proposer. Two shapes reach this +//! path, each routed through the existing Calvin entry point that builds a +//! VALID write set for it: //! -//! - A **point write** whose key a pending commit holds (Document / KV / Vector / -//! single-home edge): submitted as a single-vshard -//! [`build_single_vshard_tx_class`] + [`submit_calvin_routed`]. Because it -//! targets one vshard, it uses the single-vshard opt-in rather than the strict -//! multi-vshard builder. The scheduler acquires its key on the SAME lock -//! table the gate probed, queues FIFO behind the holder, and applies it once -//! released. +//! - A **write with a static write set** whose key a pending commit holds (a +//! row write, a collection-wide write, an edge on several homes): submitted +//! as a single-vshard [`build_single_vshard_tx_class`] + +//! [`submit_calvin_routed`]. A write on one vshard uses the single-vshard +//! opt-in rather than the strict multi-vshard builder. The scheduler +//! acquires its keys on the SAME lock table the gate probed, queues FIFO +//! behind the holder, and applies it once released. //! - A **predicate write** (`BulkUpdate` / `BulkDelete` on a SINGLE //! collection): its write set is not statically known, so it goes through //! [`dispatch_dependent_edge_recon`], which runs the pre-exec @@ -42,6 +43,7 @@ use crate::control::planner::calvin::{ submit_calvin_routed, }; use crate::control::state::SharedState; +use crate::event::EventSource; use crate::types::{DatabaseId, RequestId, TenantId, VShardId}; use nodedb_physical::physical_plan::PhysicalPlan; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; @@ -65,10 +67,23 @@ pub fn bare_ok_response(request_id: RequestId) -> Response { } } +/// Whether the scheduler applies `plan` exactly as its writer asked when the +/// gate routes it. A static write set carries `event_source`. The +/// reconnaissance of a predicate write applies it as a client write. No +/// route carries a restore id. A write the route cannot keep waits for its +/// keys at the gate instead. +pub fn calvin_route_keeps(plan: &PhysicalPlan, event_source: EventSource, restore_id: u64) -> bool { + restore_id == 0 && (matches!(event_source, EventSource::User) || !is_dependent_predicate(plan)) +} + /// Route one write to the deterministic scheduler and return the applied /// `Response`. `None` for a plain write with no RETURNING rows — the caller then /// synthesizes its normal command-tag response. /// +/// A static write set applies under `event_source` on every replica. The +/// gate routes a predicate write only when it is a client's (see +/// [`calvin_route_keeps`]). +/// /// Returns a boxed future (rather than an `async fn`) to break the async /// recursion cycle described in the module docs. pub fn route_write_to_calvin<'a>( @@ -77,6 +92,7 @@ pub fn route_write_to_calvin<'a>( database_id: DatabaseId, vshard_id: VShardId, plan: PhysicalPlan, + event_source: EventSource, ) -> Pin>> + Send + 'a>> { Box::pin(async move { let task = PhysicalTask { @@ -101,11 +117,11 @@ pub fn route_write_to_calvin<'a>( return Ok(recon.apply_result); } - // Point write: its key is known, so build a static TxClass and submit it. + // A static write set: build a static TxClass and submit it. // - // This write reaches here ONLY because `admit` returned `RouteToCalvin`: - // a pending commit already holds its key. A point write targets a single - // vshard, so it must be built with the single-vshard opt-in — it sequences + // This write reaches here ONLY because a gate returned `RouteToCalvin`: + // a pending commit already holds one of its keys. A write on a single + // vshard must be built with the single-vshard opt-in — it sequences // through the scheduler to serialize on the SAME shared per-vShard // `LockManager` the holder is on, rather than being rejected as a // (spuriously) single-vshard multi-shard dispatch. @@ -124,7 +140,8 @@ pub fn route_write_to_calvin<'a>( Some(task) => crate::control::write_resolve::resolved_ingest_counts(&task.plan)?, None => None, }; - let tx_class = build_single_vshard_tx_class(&tasks, tenant_id, &[])?; + let mut tx_class = build_single_vshard_tx_class(&tasks, tenant_id, &[])?; + tx_class.set_event_source(event_source.wal_code()); let applied = submit_calvin_routed(shared, tx_class).await?; Ok(with_ingest_counts(applied, counts)) }) diff --git a/nodedb/src/control/server/shared/write_admission/wait.rs b/nodedb/src/control/server/shared/write_admission/wait.rs new file mode 100644 index 000000000..73cb22eac --- /dev/null +++ b/nodedb/src/control/server/shared/write_admission/wait.rs @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A contended write no Calvin transaction can sequence: it waits for its +//! keys at the gate. +//! +//! The waiter holds no key while it waits. It probes the lock table with an +//! all-or-nothing `try_acquire` after every release, so it never holds one +//! key while it waits for another. A Calvin transaction therefore never +//! waits on a waiting writer, and the two cannot deadlock. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use tokio::sync::mpsc::UnboundedSender; + +use crate::control::cluster::calvin::scheduler::lock_manager::{ + LockKey, LockManager, LockMode, TxnId, +}; +use crate::types::RequestId; + +use super::gate::WriteAdmissionGuard; + +/// The keys of a contended write, and the lock table it waits on. +pub struct AdmissionWait { + lock_manager: Arc>, + txn: TxnId, + keys: BTreeMap, + promotion_sender: Option>>, +} + +impl AdmissionWait { + pub(super) fn new( + lock_manager: Arc>, + txn: TxnId, + keys: BTreeMap, + promotion_sender: Option>>, + ) -> Self { + Self { + lock_manager, + txn, + keys, + promotion_sender, + } + } + + /// Wait until every key is free, take them all, and return the guard + /// that holds them. + /// + /// Returns [`crate::Error::DeadlineExceeded`] when the keys stay held + /// until `deadline`. The write then holds nothing. + pub async fn acquire( + self, + deadline: tokio::time::Instant, + ) -> crate::Result { + let signal = self + .lock_manager + .lock() + .unwrap_or_else(|p| p.into_inner()) + .release_signal(); + loop { + // Enabled before the probe, so a release between the probe and + // the wait still wakes this waiter. + let released = signal.notified(); + tokio::pin!(released); + released.as_mut().enable(); + let acquired = { + let mut lock_manager = self.lock_manager.lock().unwrap_or_else(|p| p.into_inner()); + lock_manager.try_acquire(self.txn, self.keys.clone()) + }; + if acquired { + return Ok(WriteAdmissionGuard::new( + self.lock_manager, + self.txn, + self.promotion_sender, + )); + } + if tokio::time::timeout_at(deadline, released).await.is_err() { + return Err(crate::Error::DeadlineExceeded { + request_id: RequestId::new(0), + }); + } + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + fn coll() -> LockKey { + LockKey::Collection { + collection: Arc::from("c"), + } + } + + fn waiter(lock_manager: &Arc>) -> AdmissionWait { + AdmissionWait::new( + Arc::clone(lock_manager), + TxnId::new(TxnId::AUTOCOMMIT_EPOCH, 1), + BTreeMap::from([(coll(), LockMode::Exclusive)]), + None, + ) + } + + /// The waiter takes its keys once the Calvin holder releases them. + #[tokio::test] + async fn a_waiter_takes_its_keys_after_the_holder_releases() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let holder = TxnId::new(7, 0); + assert!( + lock_manager + .lock() + .expect("lock table") + .try_acquire(holder, BTreeMap::from([(coll(), LockMode::Intent)])) + ); + let wait = tokio::spawn( + waiter(&lock_manager).acquire(tokio::time::Instant::now() + Duration::from_secs(10)), + ); + tokio::task::yield_now().await; + assert!( + !wait.is_finished(), + "the waiter waits while the key is held" + ); + lock_manager.lock().expect("lock table").release(holder); + let guard = wait.await.expect("join").expect("the keys free up"); + drop(guard); + assert_eq!(lock_manager.lock().expect("lock table").lock_count(), 0); + } + + /// The waiter gives up at its deadline and holds nothing. + #[tokio::test] + async fn a_waiter_gives_up_at_its_deadline() { + let lock_manager = Arc::new(Mutex::new(LockManager::new())); + let holder = TxnId::new(7, 0); + assert!( + lock_manager + .lock() + .expect("lock table") + .try_acquire(holder, BTreeMap::from([(coll(), LockMode::Exclusive)])) + ); + let outcome = waiter(&lock_manager) + .acquire(tokio::time::Instant::now() + Duration::from_millis(20)) + .await; + assert!(matches!( + outcome, + Err(crate::Error::DeadlineExceeded { .. }) + )); + assert_eq!(lock_manager.lock().expect("lock table").holder_count(), 1); + } +} diff --git a/nodedb/src/control/server/shuffle/aggregator_hook.rs b/nodedb/src/control/server/shuffle/aggregator_hook.rs index 9883b0f89..c55267f1e 100644 --- a/nodedb/src/control/server/shuffle/aggregator_hook.rs +++ b/nodedb/src/control/server/shuffle/aggregator_hook.rs @@ -66,8 +66,7 @@ const SIDE_PRODUCER: u8 = 0; /// SINGLE-SIDED aggregate sibling of [`RegistryShuffleConsumer`]. Holds the /// node's [`SharedState`] so it can reach the shuffle receiver registry (for the /// staged inbox), the SPSC dispatcher + request tracker (to run the merge + -/// finalize on the Data Plane), and the network tuning (deadline / result-byte -/// ceilings). +/// finalize on the Data Plane), and the network tuning (result-byte ceiling). pub struct RegistryShuffleAggregator { state: Arc, } @@ -107,6 +106,7 @@ impl RegistryShuffleAggregator { // deadline; on expiry surface a deterministic DeadlineExceeded rather // than hanging. let deadline_ms = req.deadline_remaining_ms.max(1); + let hop_deadline = Instant::now() + Duration::from_millis(deadline_ms); if tokio::time::timeout(Duration::from_millis(deadline_ms), inbox.wait_finalized()) .await .is_err() @@ -160,9 +160,9 @@ impl RegistryShuffleAggregator { // bounded-collect path the join consume hook uses; the plan is built // locally (node-local path, never wire-encoded), so we build the // `Request` directly rather than round-tripping through `plan_bytes`. - let deadline = Duration::from_millis(deadline_ms).min(Duration::from_secs( - self.state.tuning.network.default_deadline_secs, - )); + // The hop runs on what is left of the budget the coordinator sent. + // This node's default is no ceiling on it. + let deadline = hop_deadline.saturating_duration_since(Instant::now()); let request_id = self.state.next_request_id(); let request = Request { @@ -171,7 +171,7 @@ impl RegistryShuffleAggregator { database_id: DatabaseId::from(req.database_id), vshard_id: crate::types::VShardId::new(0), plan, - deadline: Instant::now() + deadline, + deadline: hop_deadline, priority: Priority::Normal, trace_id: nodedb_types::TraceId(req.trace_id), consistency: ReadConsistency::Strong, diff --git a/nodedb/src/control/server/shuffle/consumer_hook.rs b/nodedb/src/control/server/shuffle/consumer_hook.rs index a6f11b563..15a1d3d3d 100644 --- a/nodedb/src/control/server/shuffle/consumer_hook.rs +++ b/nodedb/src/control/server/shuffle/consumer_hook.rs @@ -61,8 +61,8 @@ const SIDE_PROBE: u8 = 1; /// /// Holds the node's [`SharedState`] so it can reach the shuffle receiver /// registry (for the staged inboxes), the SPSC dispatcher + request tracker (to -/// run the grace join on the Data Plane), and the network tuning (deadline / -/// result-byte ceilings). +/// run the grace join on the Data Plane), and the network tuning (result-byte +/// ceiling). pub struct RegistryShuffleConsumer { state: Arc, } @@ -110,6 +110,7 @@ impl RegistryShuffleConsumer { // individually — is capped; on expiry we surface a deterministic // DeadlineExceeded rather than hanging. let deadline_ms = req.deadline_remaining_ms.max(1); + let hop_deadline = Instant::now() + Duration::from_millis(deadline_ms); let wait_both = async { build_inbox.wait_finalized().await; probe_inbox.wait_finalized().await; @@ -162,9 +163,9 @@ impl RegistryShuffleConsumer { // plan is built locally (it carries node-local paths and must never be // wire-encoded), so we build the `Request` directly rather than // round-tripping through `plan_bytes`. - let deadline = Duration::from_millis(deadline_ms).min(Duration::from_secs( - self.state.tuning.network.default_deadline_secs, - )); + // The hop runs on what is left of the budget the coordinator sent. + // This node's default is no ceiling on it. + let deadline = hop_deadline.saturating_duration_since(Instant::now()); let request_id = self.state.next_request_id(); let request = Request { @@ -173,7 +174,7 @@ impl RegistryShuffleConsumer { database_id: DatabaseId::from(req.database_id), vshard_id: crate::types::VShardId::new(0), plan, - deadline: Instant::now() + deadline, + deadline: hop_deadline, priority: Priority::Normal, trace_id: nodedb_types::TraceId(req.trace_id), consistency: ReadConsistency::Strong, diff --git a/nodedb/src/control/server/surrogate_exchange/authority.rs b/nodedb/src/control/server/surrogate_exchange/authority.rs index 71e870ecc..fc0420d10 100644 --- a/nodedb/src/control/server/surrogate_exchange/authority.rs +++ b/nodedb/src/control/server/surrogate_exchange/authority.rs @@ -78,7 +78,8 @@ pub(crate) async fn assign_many_at_home( home.as_u32(), ReplicatedWrite::SurrogateBind { identities }, ); - propose_replicated_entry(state, proposer, entry).await?; + let deadline = crate::control::wal_replication::statement_propose_deadline(state); + propose_replicated_entry(state, proposer, entry, deadline).await?; } pks.iter() .map(|pk| { diff --git a/nodedb/src/control/state/calvin_local.rs b/nodedb/src/control/state/calvin_local.rs index c8d0e0bb3..974363ad5 100644 --- a/nodedb/src/control/state/calvin_local.rs +++ b/nodedb/src/control/state/calvin_local.rs @@ -10,6 +10,7 @@ use std::sync::{Arc, Mutex, OnceLock}; use crate::control::cluster::calvin::scheduler::SequencerProposer; use crate::control::cluster::calvin::scheduler::lock::HotKeyTable; use crate::control::cluster::calvin::scheduler::lock_manager::{LockManager, TxnId}; +use crate::control::server::shared::write_admission::AdmissionHolds; use super::calvin_apply_sidecar::CalvinApplySidecar; use super::calvin_bases::CalvinBases; @@ -73,6 +74,9 @@ pub struct CalvinLocalState { /// with [`TxnId::AUTOCOMMIT_EPOCH`] to mint holder identities that never /// collide with a real Calvin `(epoch, position)` schedule position. pub autocommit_lock_seq: AtomicU32, + /// The lock keys replicated writes hold on this data-group leader, from + /// their propose until the apply loop starts their entries. + pub admission_holds: AdmissionHolds, /// The backup cut markers each local scheduler passed. pub cuts: CalvinCuts, /// Where this node's Calvin state of each vShard is whole from. @@ -98,6 +102,7 @@ impl CalvinLocalState { hot_key_table: Arc::new(Mutex::new(HotKeyTable::new())), promotion_senders: Arc::new(Mutex::new(BTreeMap::new())), autocommit_lock_seq: AtomicU32::new(0), + admission_holds: AdmissionHolds::new(), cuts: CalvinCuts::default(), bases: CalvinBases::default(), sequencer_proposer: OnceLock::new(), diff --git a/nodedb/src/control/wal_replication/decode/entry.rs b/nodedb/src/control/wal_replication/decode/entry.rs index 5878670ed..aafc820ad 100644 --- a/nodedb/src/control/wal_replication/decode/entry.rs +++ b/nodedb/src/control/wal_replication/decode/entry.rs @@ -55,6 +55,15 @@ fn decode_with_database(data: &[u8]) -> crate::Result e, None => return Ok(None), }; + decode_parsed_entry(&entry) +} + +/// Decodes an entry already parsed from its bytes into its database and its +/// plan, without binding any identity. `None` for an array CRDT op or array +/// schema, which the distributed applier handles before any plan exists. +pub fn decode_parsed_entry( + entry: &ReplicatedEntry, +) -> crate::Result> { // Array CRDT variants are handled by the distributed applier before this call. match &entry.write { ReplicatedWrite::ArrayOp { .. } | ReplicatedWrite::ArraySchema { .. } => { diff --git a/nodedb/src/control/wal_replication/decode/mod.rs b/nodedb/src/control/wal_replication/decode/mod.rs index bcbd9f2ec..f658d42b0 100644 --- a/nodedb/src/control/wal_replication/decode/mod.rs +++ b/nodedb/src/control/wal_replication/decode/mod.rs @@ -38,6 +38,6 @@ mod transaction_redo; mod vector; mod vector_direct; -pub use entry::{decode_replicated_entry, from_replicated_entry}; +pub use entry::{decode_parsed_entry, decode_replicated_entry, from_replicated_entry}; pub use entry_graph::edge_write_plan; pub use transaction_redo::transaction_redo_payload; diff --git a/nodedb/src/control/wal_replication/mod.rs b/nodedb/src/control/wal_replication/mod.rs index b232b0a52..184ddb2e1 100644 --- a/nodedb/src/control/wal_replication/mod.rs +++ b/nodedb/src/control/wal_replication/mod.rs @@ -19,6 +19,7 @@ pub use decode::{decode_replicated_entry, from_replicated_entry}; pub use encode::to_replicated_entry; pub(crate) use propose::{ propose_replicated_entry, stamp_collection_incarnations, stamp_metadata_floor, + statement_propose_deadline, }; pub use replicable_write::ReplicableWrite; pub use types::{ diff --git a/nodedb/src/control/wal_replication/propose.rs b/nodedb/src/control/wal_replication/propose.rs index 1a57507d9..13cbba08a 100644 --- a/nodedb/src/control/wal_replication/propose.rs +++ b/nodedb/src/control/wal_replication/propose.rs @@ -19,6 +19,14 @@ const FIRST_BACKOFF: std::time::Duration = std::time::Duration::from_millis(10); /// Longest wait between two re-proposals. const MAX_BACKOFF: std::time::Duration = std::time::Duration::from_millis(200); +/// The deadline a proposal made now carries: the running statement's +/// deadline, or the node default outside a statement. +pub(crate) fn statement_propose_deadline(state: &SharedState) -> tokio::time::Instant { + tokio::time::Instant::from_std(crate::control::server::shared::session::statement_deadline( + state.tuning.network.default_deadline_secs, + )) +} + /// Stamp this node's metadata floor on `entry`: the catalog the write was /// planned against, including the batch being applied now (see /// `AppliedIndexWatcher::floor`). Every replica applies the entry only once @@ -58,8 +66,11 @@ pub(crate) fn stamp_collection_incarnations( /// (`planner::calvin::edge_sequencing`), and its applied payload and read /// version come back the same way. /// -/// Re-proposes the same payload until the statement deadline while the group -/// has no leader to take it: +/// `deadline` is the caller's deadline. Every attempt, each attempt's wait at +/// the leader's write gate, and each wait for the local apply end at it. +/// +/// Re-proposes the same payload until `deadline` while the group has no +/// leader to take it: /// - [`crate::Error::RetryableLeaderChange`]: a new leader's election no-op /// overwrote the previous leader's entry; /// - [`crate::Error::NoLeader`]: the group is electing, or a leadership @@ -73,6 +84,7 @@ pub(crate) async fn propose_replicated_entry( state: &SharedState, proposer: &Arc, mut entry: ReplicatedEntry, + deadline: tokio::time::Instant, ) -> crate::Result<(Vec, crate::types::Lsn)> { // An edge write runs as a Calvin transaction, never as a data-group // entry, so every edge version of a collection takes a Calvin ordinal. @@ -91,10 +103,6 @@ pub(crate) async fn propose_replicated_entry( let data = entry.encode()?; let vshard_id = entry.vshard_id; - // The statement deadline. Every attempt, and each attempt's wait for the - // local apply, ends at this one instant. - let deadline = tokio::time::Instant::now() - + std::time::Duration::from_secs(state.tuning.network.default_deadline_secs); let mut backoff = FIRST_BACKOFF; let mut attempt: u32 = 0; let payload = loop { diff --git a/nodedb/src/control/write_resolve/propose.rs b/nodedb/src/control/write_resolve/propose.rs index 90a8eecad..e06b3224f 100644 --- a/nodedb/src/control/write_resolve/propose.rs +++ b/nodedb/src/control/write_resolve/propose.rs @@ -42,7 +42,8 @@ pub(super) async fn propose_resolved( ), })?; - match propose_replicated_entry(state, proposer, entry).await { + let deadline = crate::control::wal_replication::statement_propose_deadline(state); + match propose_replicated_entry(state, proposer, entry, deadline).await { Ok((payload, write_version)) => { let request_id = RequestId::new(state.request_id_counter.fetch_add(1, Ordering::Relaxed)); diff --git a/nodedb/src/event/topic/publish.rs b/nodedb/src/event/topic/publish.rs index fce03896c..04d18d751 100644 --- a/nodedb/src/event/topic/publish.rs +++ b/nodedb/src/event/topic/publish.rs @@ -100,15 +100,19 @@ async fn publish_replicated( entry: ReplicatedEntry, topic_name: &str, ) -> Result { - let (payload, _) = - crate::control::wal_replication::propose_replicated_entry(state, proposer, entry) - .await - .map_err(|error| match error { - crate::Error::UndefinedObject { kind: "topic", .. } => { - PublishError::TopicNotFound(topic_name.to_string()) - } - other => PublishError::Persistence(other.to_string()), - })?; + let (payload, _) = crate::control::wal_replication::propose_replicated_entry( + state, + proposer, + entry, + crate::control::wal_replication::statement_propose_deadline(state), + ) + .await + .map_err(|error| match error { + crate::Error::UndefinedObject { kind: "topic", .. } => { + PublishError::TopicNotFound(topic_name.to_string()) + } + other => PublishError::Persistence(other.to_string()), + })?; zerompk::from_msgpack::(&payload) .map_err(|error| PublishError::Persistence(format!("publish result: {error}"))) } diff --git a/nodedb/tests/inproc/cases/mod.rs b/nodedb/tests/inproc/cases/mod.rs index 6bd9f9821..1dbd62460 100644 --- a/nodedb/tests/inproc/cases/mod.rs +++ b/nodedb/tests/inproc/cases/mod.rs @@ -214,4 +214,5 @@ mod wal_catchup; mod warm_storage_object_store; mod wire_server_version; mod write_admission_fence; +mod write_admission_leader_gate; mod ws_live_lifecycle; diff --git a/nodedb/tests/inproc/cases/write_admission_fence.rs b/nodedb/tests/inproc/cases/write_admission_fence.rs index 2a34d4148..1d380e298 100644 --- a/nodedb/tests/inproc/cases/write_admission_fence.rs +++ b/nodedb/tests/inproc/cases/write_admission_fence.rs @@ -36,7 +36,7 @@ use nodedb_types::QualifiedCollection; /// Build a single-node `SharedState` over a throwaway WAL. The returned /// `TempDir` must be kept alive for the WAL's lifetime. -fn build_shared() -> (Arc, tempfile::TempDir) { +pub(super) fn build_shared() -> (Arc, tempfile::TempDir) { let dir = tempfile::tempdir().expect("tempdir"); let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("fence.wal")).expect("open wal")); @@ -47,7 +47,7 @@ fn build_shared() -> (Arc, tempfile::TempDir) { /// Register an empty lock table for `collection`'s vShard and return both the /// shared `Arc` and the vShard the gate will resolve the plan to. -fn register_lock_manager( +pub(super) fn register_lock_manager( shared: &SharedState, collection: &str, ) -> (Arc>, VShardId) { @@ -65,7 +65,7 @@ fn register_lock_manager( /// Register a promotion channel for `vshard` and return the receiver. The gate /// clones the sender into any fast-path guard it builds for this vShard, so the /// guard's drop delivers promoted scheduler waiters here. -fn register_promotion_channel( +pub(super) fn register_promotion_channel( shared: &SharedState, vshard: VShardId, ) -> tokio::sync::mpsc::UnboundedReceiver> { @@ -80,7 +80,7 @@ fn register_promotion_channel( } /// A single-key KV point write to `collection` / `key`. -fn kv_put(collection: &str, key: &[u8]) -> PhysicalPlan { +pub(super) fn kv_put(collection: &str, key: &[u8]) -> PhysicalPlan { PhysicalPlan::Kv(KvOp::Put { collection: QualifiedCollection::new(DatabaseId::DEFAULT, collection), key: key.to_vec(), @@ -131,7 +131,7 @@ async fn fence_write_blocks_behind_held_commit_lock() { // holder via the scheduler's FIFO queue. let plan = kv_put(coll, b"K"); let before = cp_routed_to_calvin(); - match admit(&shared, &target(vshard, &plan)) { + match admit(&shared, &target(vshard, &plan)).expect("admit") { WriteAdmission::RouteToCalvin => {} _ => panic!("a point write behind a held commit lock must route to Calvin"), } @@ -142,7 +142,7 @@ async fn fence_write_blocks_behind_held_commit_lock() { // The commit releases; the same write is now admitted to the fast path. let _ = lm.lock().expect("lm").release(commit_txn); - match admit(&shared, &target(vshard, &plan)) { + match admit(&shared, &target(vshard, &plan)).expect("admit") { WriteAdmission::FastPath { guard: Some(_) } => {} _ => panic!("after release, the point write must fast-path with a real lock guard"), } @@ -159,14 +159,14 @@ async fn two_concurrent_same_key_point_writes_serialize() { let plan = kv_put(coll, b"K"); // First write takes the fast path and holds the lock via its RAII guard. - let guard1 = match admit(&shared, &target(vshard, &plan)) { + let guard1 = match admit(&shared, &target(vshard, &plan)).expect("admit") { WriteAdmission::FastPath { guard: Some(g) } => g, _ => panic!("first same-key write must fast-path with a real lock guard"), }; // Second write to the same key observes the held lock and is routed. let before = cp_routed_to_calvin(); - match admit(&shared, &target(vshard, &plan)) { + match admit(&shared, &target(vshard, &plan)).expect("admit") { WriteAdmission::RouteToCalvin => {} _ => panic!("second same-key write must route to Calvin behind the first"), } @@ -177,7 +177,7 @@ async fn two_concurrent_same_key_point_writes_serialize() { // First write completes (guard drops, releasing the lock); the key is free. drop(guard1); - match admit(&shared, &target(vshard, &plan)) { + match admit(&shared, &target(vshard, &plan)).expect("admit") { WriteAdmission::FastPath { guard: Some(_) } => {} _ => panic!("after the first write releases, the key must fast-path again"), } @@ -195,7 +195,7 @@ async fn single_node_point_write_uses_global_keyed_order_lock() { let vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, coll).vshard(); let plan = kv_put(coll, b"K"); - let lock = match admit(&shared, &target(vshard, &plan)) { + let lock = match admit(&shared, &target(vshard, &plan)).expect("admit") { WriteAdmission::FastPathBlocking { key, keyed_lock } => { assert_eq!( key, @@ -222,7 +222,7 @@ async fn single_node_same_key_serializes_fifo() { let vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, coll).vshard(); let plan = kv_put(coll, b"K"); - let (key, lock) = match admit(&shared, &target(vshard, &plan)) { + let (key, lock) = match admit(&shared, &target(vshard, &plan)).expect("admit") { WriteAdmission::FastPathBlocking { key, keyed_lock } => (key, keyed_lock), _ => panic!("single-node point write must return FastPathBlocking"), }; @@ -281,7 +281,7 @@ async fn fast_path_drop_delivers_promoted_scheduler_txn() { // A fast-path point write on K takes the fence via its RAII guard (an // autocommit-band holder). K was uncontended at acquire time. let plan = kv_put(coll, b"K"); - let guard = match admit(&shared, &target(vshard, &plan)) { + let guard = match admit(&shared, &target(vshard, &plan)).expect("admit") { WriteAdmission::FastPath { guard: Some(g) } => g, _ => panic!("uncontended point write must fast-path with a real lock guard"), }; @@ -340,3 +340,93 @@ async fn single_node_distinct_keys_do_not_block() { drop(g_b); drop(g_a); } + +/// The collection lock key a truncate holds `Exclusive` and a row writer +/// holds `Intent`. +fn collection_lock_key(collection: &str) -> LockKey { + LockKey::Collection { + collection: Arc::from(collection), + } +} + +/// A Calvin truncate and an autocommit point write of one collection +/// serialize at the gate, whichever comes first. A truncate holds the +/// collection `Exclusive`, and a point write holds it `Intent`. +#[tokio::test] +async fn calvin_truncate_and_autocommit_point_write_serialize() { + let (shared, _dir) = build_shared(); + let coll = "truncate_coll"; + let (lm, vshard) = register_lock_manager(&shared, coll); + let mut promotion_rx = register_promotion_channel(&shared, vshard); + let truncate_keys: BTreeMap = + [(collection_lock_key(coll), LockMode::Exclusive)].into(); + let plan = kv_put(coll, b"K"); + + // The truncate is sequenced first: the point write routes behind it. + let truncate = TxnId::new(9, 0); + assert_eq!( + lm.lock() + .expect("lm") + .acquire(truncate, truncate_keys.clone()), + AcquireOutcome::Ready + ); + assert!( + matches!( + admit(&shared, &target(vshard, &plan)).expect("admit"), + WriteAdmission::RouteToCalvin + ), + "a point write of a collection a truncate holds must route behind it" + ); + let _ = lm.lock().expect("lm").release(truncate); + + // The point write is admitted first: the truncate waits for its guard. + let guard = match admit(&shared, &target(vshard, &plan)).expect("admit") { + WriteAdmission::FastPath { guard: Some(guard) } => guard, + _ => panic!("an uncontended point write must fast-path with a real lock guard"), + }; + let later_truncate = TxnId::new(9, 1); + assert_eq!( + lm.lock() + .expect("lm") + .acquire(later_truncate, truncate_keys.clone()), + AcquireOutcome::Blocked, + "a truncate must wait for the point write's collection intent" + ); + drop(guard); + let promoted = promotion_rx + .try_recv() + .expect("the guard's drop must deliver the promoted truncate"); + assert!(promoted.contains(&later_truncate)); + assert!( + lm.lock() + .expect("lm") + .is_ready(later_truncate, &truncate_keys) + ); +} + +/// An autocommit truncate locks its collection `Exclusive`: it routes +/// behind a Calvin row writer of the collection. +#[tokio::test] +async fn autocommit_truncate_routes_behind_a_calvin_row_writer() { + let (shared, _dir) = build_shared(); + let coll = "truncate_after_row"; + let (lm, vshard) = register_lock_manager(&shared, coll); + let row_writer = TxnId::new(4, 0); + let row_keys: BTreeMap = [ + (kv_lock_key(coll, b"K"), LockMode::Exclusive), + (collection_lock_key(coll), LockMode::Intent), + ] + .into(); + assert_eq!( + lm.lock().expect("lm").acquire(row_writer, row_keys), + AcquireOutcome::Ready + ); + let truncate = PhysicalPlan::Kv(KvOp::Truncate { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), + restart_identity: false, + }); + assert!(matches!( + admit(&shared, &target(vshard, &truncate)).expect("admit"), + WriteAdmission::RouteToCalvin + )); +} diff --git a/nodedb/tests/inproc/cases/write_admission_leader_gate.rs b/nodedb/tests/inproc/cases/write_admission_leader_gate.rs new file mode 100644 index 000000000..d20016061 --- /dev/null +++ b/nodedb/tests/inproc/cases/write_admission_leader_gate.rs @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The data-group leader's write gate. +//! +//! A replicated write takes its lock keys on its leader before its propose. +//! These tests drive `LeaderWriteGate` with encoded entries against the lock +//! table a Calvin scheduler shares: +//! +//! - An uncontended entry holds its keys until the leader's apply loop starts +//! the entry. A Calvin truncate of the collection waits until then. +//! - A contended entry a Calvin transaction can sequence is refused with +//! `RouteToSequencer`. +//! - A contended entry no Calvin transaction sequences waits for its keys, +//! until the proposer's deadline at the latest. + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; + +use nodedb::control::cluster::calvin::scheduler::lock_manager::{ + AcquireOutcome, LockKey, LockMode, TxnId, +}; +use nodedb::control::server::shared::write_admission::LeaderWriteGate; +use nodedb::control::wal_replication::{ReplicableWrite, to_replicated_entry}; +use nodedb::types::{DatabaseId, TenantId, VShardId}; +use nodedb_cluster::{CalvinError, ClusterError, DataProposeGate}; +use nodedb_physical::physical_plan::{KvOp, KvResolvedMutation, PhysicalPlan}; +use nodedb_types::{QualifiedCollection, Surrogate}; + +use super::write_admission_fence::{build_shared, kv_put, register_lock_manager}; + +/// The encoded replicated entry of `plan` on `vshard`. +fn entry_bytes(vshard: VShardId, plan: &PhysicalPlan) -> Vec { + let replicable = ReplicableWrite::decide_for_replication(plan).expect("replicable"); + to_replicated_entry(TenantId::new(1), DatabaseId::DEFAULT, vshard, &replicable) + .expect("encode") + .expect("the plan has a replicated form") + .encode() + .expect("entry bytes") +} + +/// A proposer deadline no test reaches. +fn far_deadline() -> tokio::time::Instant { + tokio::time::Instant::now() + Duration::from_secs(60) +} + +fn truncate_keys(collection: &str) -> BTreeMap { + [( + LockKey::Collection { + collection: Arc::from(collection), + }, + LockMode::Exclusive, + )] + .into() +} + +/// An admitted entry holds its keys until the leader starts its apply. A +/// Calvin truncate sequenced meanwhile runs after it. +#[tokio::test] +async fn an_admitted_entry_holds_its_keys_until_its_apply_starts() { + let (shared, _dir) = build_shared(); + let coll = "leader_gate_hold"; + let (lm, vshard) = register_lock_manager(&shared, coll); + let gate = LeaderWriteGate::new(&shared); + + let hold = gate + .admit( + vshard.as_u32(), + &entry_bytes(vshard, &kv_put(coll, b"K")), + far_deadline(), + ) + .await + .expect("admit") + .expect("an uncontended row write holds its keys"); + let (group_id, log_index) = (3, 10); + hold.landed(group_id, log_index); + + let truncate = TxnId::new(9, 0); + let keys = truncate_keys(coll); + assert_eq!( + lm.lock().expect("lm").acquire(truncate, keys.clone()), + AcquireOutcome::Blocked, + "a Calvin truncate waits for the admitted write" + ); + shared + .calvin + .admission_holds + .release_through(group_id, log_index - 1); + assert!( + !lm.lock().expect("lm").is_ready(truncate, &keys), + "an earlier entry's start releases nothing of this one" + ); + shared + .calvin + .admission_holds + .release_through(group_id, log_index); + assert!( + lm.lock().expect("lm").is_ready(truncate, &keys), + "the entry's start releases its keys to the truncate" + ); +} + +/// A contended entry a Calvin transaction can sequence is refused, so its +/// proposer submits it to the sequencer. +#[tokio::test] +async fn a_contended_sequenced_entry_routes_to_the_sequencer() { + let (shared, _dir) = build_shared(); + let coll = "leader_gate_route"; + let (lm, vshard) = register_lock_manager(&shared, coll); + assert_eq!( + lm.lock() + .expect("lm") + .acquire(TxnId::new(9, 0), truncate_keys(coll)), + AcquireOutcome::Ready + ); + let gate = LeaderWriteGate::new(&shared); + let refused = gate + .admit( + vshard.as_u32(), + &entry_bytes(vshard, &kv_put(coll, b"K")), + far_deadline(), + ) + .await; + assert!(matches!( + refused, + Err(ClusterError::Calvin(CalvinError::RouteToSequencer)) + )); +} + +/// A contended resolved write, which no Calvin transaction sequences, waits +/// for its keys and takes them once the holder releases. +#[tokio::test] +async fn a_contended_resolved_entry_waits_for_its_keys() { + let (shared, _dir) = build_shared(); + let coll = "leader_gate_wait"; + let (lm, vshard) = register_lock_manager(&shared, coll); + let truncate = TxnId::new(9, 0); + assert_eq!( + lm.lock() + .expect("lm") + .acquire(truncate, truncate_keys(coll)), + AcquireOutcome::Ready + ); + let resolved = PhysicalPlan::Kv(KvOp::ResolvedWrite { + mutations: vec![KvResolvedMutation::Put { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), + key: b"K".to_vec(), + value: b"v".to_vec(), + ttl_ms: 0, + expire_at_ms: 0, + surrogate: Surrogate::new(5), + precondition: None, + }], + response_payload: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::decided_earlier_in_request(), + }); + let bytes = entry_bytes(vshard, &resolved); + + let waiter = { + let gate = LeaderWriteGate::new(&shared); + let vshard = vshard.as_u32(); + tokio::spawn(async move { gate.admit(vshard, &bytes, far_deadline()).await }) + }; + tokio::task::yield_now().await; + assert!( + !waiter.is_finished(), + "the entry waits while the truncate holds" + ); + let _ = lm.lock().expect("lm").release(truncate); + let hold = waiter + .await + .expect("join") + .expect("the keys free up") + .expect("the resolved write holds its row"); + drop(hold); + assert_eq!( + lm.lock() + .expect("lm") + .acquire(TxnId::new(10, 0), truncate_keys(coll)), + AcquireOutcome::Ready, + "dropping the hold frees every key it held" + ); +} + +/// A contended entry waits no longer than the proposer's deadline. A +/// forwarded entry carries a short budget, and the gate refuses it +/// unproposed once the budget runs out. +#[tokio::test] +async fn a_contended_entry_waits_no_longer_than_its_budget() { + let (shared, _dir) = build_shared(); + let coll = "leader_gate_budget"; + let (lm, vshard) = register_lock_manager(&shared, coll); + assert_eq!( + lm.lock() + .expect("lm") + .acquire(TxnId::new(9, 0), truncate_keys(coll)), + AcquireOutcome::Ready + ); + let resolved = PhysicalPlan::Kv(KvOp::ResolvedWrite { + mutations: vec![KvResolvedMutation::Put { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), + key: b"K".to_vec(), + value: b"v".to_vec(), + ttl_ms: 0, + expire_at_ms: 0, + surrogate: Surrogate::new(5), + precondition: None, + }], + response_payload: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::decided_earlier_in_request(), + }); + let budget_ms = 50; + let deadline = nodedb_cluster::rpc_codec::forwarded_deadline(budget_ms).expect("a live budget"); + let gate = LeaderWriteGate::new(&shared); + // The node default deadline is seconds long. The gate must end at the + // budget, so this bound fails a gate that waits for the default. + let refused = tokio::time::timeout( + Duration::from_secs(5), + gate.admit(vshard.as_u32(), &entry_bytes(vshard, &resolved), deadline), + ) + .await + .expect("the gate ends at the forwarded budget, not the node default"); + assert!(matches!( + refused, + Err(ClusterError::Calvin(CalvinError::AdmissionTimedOut)) + )); + assert!( + tokio::time::Instant::now() >= deadline, + "the gate waits until the budget runs out" + ); +} From d357a52bf1139eeffa31e1f873daccd46d050ac2 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Fri, 9 Oct 2026 19:34:33 +0800 Subject: [PATCH 04/19] feat(calvin): add an applied ledger, chunked redo streams, and a metadata DDL state split Calvin applied ledger and Data-Plane install - The applied mirror becomes an applied ledger in the scheduler. The Data Plane installs a Calvin transaction's redo and records it applied through a dedicated install path, and the snapshot capture and install steps carry the ledger. Chunked redo streams - A committed redo larger than max_redo_entry_bytes travels in the data-group log as chunk entries followed by a final entry. The proposer can abandon a stream, and the apply loop reassembles chunks. - A node bounds the bytes of open streams with max_open_redo_bytes, and a group leader refuses a new stream that would pass it. - The WAL appends redo chunks and holds a floor on truncation while a stream is open, so crash recovery can resume or discard it. A snapshot owed record keeps a stream's floor across restarts. - Raft limits the entry data of one AppendEntries request to MAX_APPEND_BATCH_BYTES. An entry larger than the cap ships alone. Metadata DDL state split - The pending DDL and metadata applier host state move out of the shared state fields into a metadata_ddl module. --- .../cases/ddl_prepare_lease_reclaim.rs | 6 +- .../src/physical_plan/calvin_reply.rs | 173 ++++++ .../src/physical_plan/collection.rs | 1 + nodedb-physical/src/physical_plan/meta.rs | 9 + nodedb-physical/src/physical_plan/mod.rs | 5 + nodedb-raft/src/log.rs | 40 ++ nodedb-raft/src/node/internal.rs | 12 +- nodedb-test-support/src/tx_commit.rs | 1 + nodedb-types/src/config/tuning/calvin.rs | 26 + nodedb-wal/src/record/types.rs | 13 + nodedb/src/bootstrap/calvin_catch_up.rs | 2 +- nodedb/src/config/server/config.rs | 49 ++ nodedb/src/config/server/domain.rs | 39 +- nodedb/src/control/backup/restore/sections.rs | 2 + nodedb/src/control/checkpoint_manager.rs | 15 +- nodedb/src/control/checkpoint_task.rs | 4 +- .../calvin/scheduler/applied_ledger.rs | 326 +++++++++++ .../calvin/scheduler/applied_mirror.rs | 195 ------- .../scheduler/driver/core/commit_redo.rs | 14 +- .../driver/core/commit_resolve/verdict.rs | 7 +- .../calvin/scheduler/driver/core/process.rs | 28 +- .../calvin/scheduler/driver/core/scheduler.rs | 81 +-- .../scheduler/driver/core/test_support.rs | 16 +- .../control/cluster/calvin/scheduler/mod.rs | 6 +- .../cluster/calvin/scheduler/recovery.rs | 243 +++++--- .../cluster/calvin_snapshot/capture.rs | 4 +- .../cluster/calvin_snapshot/install.rs | 17 +- .../cluster/calvin_snapshot/requirement.rs | 19 +- .../cluster/metadata_applier/boot_seed.rs | 11 +- .../cluster/metadata_applier/dispatch.rs | 8 +- .../cluster/metadata_applier/host_state.rs | 6 +- .../cluster/metadata_applier/pending_ddl.rs | 5 +- nodedb/src/control/cluster/mod.rs | 1 + .../control/cluster/redo_stream_membership.rs | 92 +++ .../src/control/cluster/snapshot_applier.rs | 23 + .../src/control/cluster/snapshot_builder.rs | 3 + .../control/cluster/snapshot_install/error.rs | 3 + .../control/cluster/snapshot_install/split.rs | 4 + nodedb/src/control/cluster/start_raft/core.rs | 3 + .../src/control/cluster/start_raft/hooks.rs | 23 + .../cluster/start_raft/proposer_wiring.rs | 8 +- .../src/control/cluster/start_raft_helpers.rs | 41 +- .../distributed_applier/applied_index.rs | 99 +++- .../distributed_applier/apply_loop/lane.rs | 68 ++- .../distributed_applier/apply_loop/mod.rs | 3 + .../apply_loop/pipeline.rs | 13 +- .../apply_loop/redo_chunk.rs | 403 +++++++++++++ .../apply_loop/snapshot_gate.rs | 83 ++- .../distributed_applier/apply_loop/start.rs | 3 + .../apply_loop/transaction_redo.rs | 113 +++- nodedb/src/control/gateway/router.rs | 1 + .../maintenance/clone_materializer/chained.rs | 16 +- .../src/control/metadata_proposer/catalog.rs | 2 +- .../control/metadata_proposer/ddl_owner.rs | 6 +- .../control/metadata_proposer/ddl_prepare.rs | 8 +- .../src/control/security/auth_fence/state.rs | 9 - .../security/auth_lease/calvin_acks.rs | 47 +- .../control/security/auth_lease/coverage.rs | 2 +- .../security/catalog/bootstrap_tables.rs | 1 + .../security/catalog/calvin_applied.rs | 41 +- nodedb/src/control/security/catalog/mod.rs | 1 + .../security/catalog/redo_snapshot_owed.rs | 104 ++++ .../security/catalog/replicated_image.rs | 2 + .../dispatch_utils/durability_barrier.rs | 1 + .../server/exchange/all_cores/snapshot.rs | 1 + .../src/control/server/ilp_batch/dispatch.rs | 2 +- .../shared/session/commit/single_shard.rs | 20 +- .../server/shared/session/ddl_compensate.rs | 2 +- .../server/shared/session/ddl_flush.rs | 6 +- .../shared/write_admission/leader_gate.rs | 58 +- .../src/control/server/wal_dispatch/core.rs | 1 + nodedb/src/control/state/calvin_local.rs | 12 +- nodedb/src/control/state/fields.rs | 18 +- nodedb/src/control/state/init.rs | 14 +- nodedb/src/control/state/init_prod/open.rs | 14 +- nodedb/src/control/state/metadata_ddl.rs | 43 ++ nodedb/src/control/state/mod.rs | 2 + .../control/wal_replication/decode/entry.rs | 7 + .../src/control/wal_replication/decode/mod.rs | 2 +- .../decode/transaction_redo.rs | 169 +++++- .../src/control/wal_replication/encode/mod.rs | 5 +- .../encode/transaction_redo.rs | 300 +++++++++- .../transaction_redo/abandon.rs | 145 +++++ .../transaction_redo/chunks/abandon.rs | 124 ++++ .../transaction_redo/chunks/carry.rs | 174 ++++++ .../transaction_redo/chunks/error.rs | 93 +++ .../transaction_redo/chunks/leave.rs | 168 ++++++ .../transaction_redo/chunks/mod.rs | 28 + .../transaction_redo/chunks/owed.rs | 118 ++++ .../transaction_redo/chunks/rebuild.rs | 274 +++++++++ .../transaction_redo/chunks/store.rs | 546 ++++++++++++++++++ .../transaction_redo/chunks/stream.rs | 56 ++ .../wal_replication/transaction_redo/mod.rs | 9 + .../transaction_redo/payload.rs | 112 +++- .../transaction_redo/propose.rs | 80 +++ .../src/control/wal_replication/types/mod.rs | 5 +- .../wal_replication/types/redo_chunk_wire.rs | 76 +++ .../wal_replication/types/replicated_write.rs | 51 +- .../types/transaction_redo_wire.rs | 24 + nodedb/src/data/executor/dispatch/meta.rs | 28 +- .../executor/handlers/control/calvin/flush.rs | 4 +- .../handlers/control/calvin/test_commit.rs | 14 +- .../handlers/control/calvin_reply/mod.rs | 1 + .../handlers/control/calvin_reply/spec.rs | 151 +++++ .../handlers/control/calvin_resolve.rs | 99 +++- .../redo_apply/calvin_fold_tests.rs | 16 +- .../transaction/redo_apply/calvin_install.rs | 94 +++ .../redo_apply/calvin_install_tests.rs | 211 +++++++ .../handlers/transaction/redo_apply/mod.rs | 6 + nodedb/src/diag/context/mod.rs | 4 + nodedb/src/diag/context/redo_stream.rs | 130 +++++ nodedb/src/diag/mod.rs | 1 + nodedb/src/diag/recording/mod.rs | 4 + nodedb/src/diag/recording/redo_stream.rs | 76 +++ nodedb/src/event/wal_replay_kind.rs | 5 +- nodedb/src/main_boot/post_open.rs | 14 +- nodedb/src/types/snapshot.rs | 8 + nodedb/src/wal/manager/append_redo_chunk.rs | 165 ++++++ nodedb/src/wal/manager/core.rs | 8 + nodedb/src/wal/manager/floor_holds.rs | 115 ++++ nodedb/src/wal/manager/mod.rs | 4 + nodedb/src/wal/manager/ops.rs | 11 +- nodedb/src/wal/mod.rs | 11 +- nodedb/src/wal/redo/chunk.rs | 182 ++++++ nodedb/src/wal/redo/mod.rs | 4 + nodedb/src/wal/replay/surrogate/dispatch.rs | 5 +- nodedb/src/wal/replay/sync_hwm/advance.rs | 3 +- nodedb/tests/crash_redo_chunk_stream.rs | 153 +++++ .../inproc/cases/calvin_executor_apply.rs | 4 +- .../cases/calvin_executor_panic_recovery.rs | 6 +- .../inproc/cases/calvin_two_phase_apply.rs | 11 +- nodedb/tests/wire/cases/mod.rs | 1 + .../wire/cases/redo_chunked_session_commit.rs | 80 +++ nodedb/tests/wire/harness/config_toml.rs | 16 + nodedb/tests/wire/harness/lifecycle.rs | 30 + 135 files changed, 6352 insertions(+), 692 deletions(-) create mode 100644 nodedb-physical/src/physical_plan/calvin_reply.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/applied_ledger.rs delete mode 100644 nodedb/src/control/cluster/calvin/scheduler/applied_mirror.rs create mode 100644 nodedb/src/control/cluster/redo_stream_membership.rs create mode 100644 nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs create mode 100644 nodedb/src/control/security/catalog/redo_snapshot_owed.rs create mode 100644 nodedb/src/control/state/metadata_ddl.rs create mode 100644 nodedb/src/control/wal_replication/transaction_redo/abandon.rs create mode 100644 nodedb/src/control/wal_replication/transaction_redo/chunks/abandon.rs create mode 100644 nodedb/src/control/wal_replication/transaction_redo/chunks/carry.rs create mode 100644 nodedb/src/control/wal_replication/transaction_redo/chunks/error.rs create mode 100644 nodedb/src/control/wal_replication/transaction_redo/chunks/leave.rs create mode 100644 nodedb/src/control/wal_replication/transaction_redo/chunks/mod.rs create mode 100644 nodedb/src/control/wal_replication/transaction_redo/chunks/owed.rs create mode 100644 nodedb/src/control/wal_replication/transaction_redo/chunks/rebuild.rs create mode 100644 nodedb/src/control/wal_replication/transaction_redo/chunks/store.rs create mode 100644 nodedb/src/control/wal_replication/transaction_redo/chunks/stream.rs create mode 100644 nodedb/src/control/wal_replication/transaction_redo/propose.rs create mode 100644 nodedb/src/control/wal_replication/types/redo_chunk_wire.rs create mode 100644 nodedb/src/data/executor/handlers/control/calvin_reply/spec.rs create mode 100644 nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install.rs create mode 100644 nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs create mode 100644 nodedb/src/diag/context/redo_stream.rs create mode 100644 nodedb/src/diag/recording/redo_stream.rs create mode 100644 nodedb/src/wal/manager/append_redo_chunk.rs create mode 100644 nodedb/src/wal/manager/floor_holds.rs create mode 100644 nodedb/src/wal/redo/chunk.rs create mode 100644 nodedb/tests/crash_redo_chunk_stream.rs create mode 100644 nodedb/tests/wire/cases/redo_chunked_session_commit.rs diff --git a/nodedb-cluster-tests/tests/common_suite/cases/ddl_prepare_lease_reclaim.rs b/nodedb-cluster-tests/tests/common_suite/cases/ddl_prepare_lease_reclaim.rs index 6282f3e7d..a932af4e5 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/ddl_prepare_lease_reclaim.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/ddl_prepare_lease_reclaim.rs @@ -22,7 +22,8 @@ const DEAD_TOKEN: u64 = 0x00dd_1ea5; fn owner_token(node: &TestClusterNode) -> Option { node.shared - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .unwrap_or_else(|p| p.into_inner()) .map(|owner| owner.token) @@ -124,7 +125,8 @@ async fn a_dead_ddl_lease_owner_is_reclaimed_before_its_lease_runs_out() { assert_ne!( survivor .shared - .metadata_ddl_applied_token + .metadata_ddl + .applied_token .load(std::sync::atomic::Ordering::Acquire), DEAD_TOKEN, "a reclaimed token's finalize applies nothing" diff --git a/nodedb-physical/src/physical_plan/calvin_reply.rs b/nodedb-physical/src/physical_plan/calvin_reply.rs new file mode 100644 index 000000000..7e61ac143 --- /dev/null +++ b/nodedb-physical/src/physical_plan/calvin_reply.rs @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The reply a committed Calvin slice answers with, in the form a redo +//! entry carries to every replica. +//! +//! The leader's stage decides the reply. Every replica renders it after its +//! own install of the slice's redo, so every replica answers the same bytes. + +use nodedb_types::Surrogate; + +use super::ReturningSpec; + +/// What a Calvin slice answers once its redo installs. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub enum CalvinReplySpec { + /// An affected count, decided when the plan staged. + Count(Vec), + /// `RETURNING` rows, decided when the plan staged. + Rows(Vec), + /// `RETURNING` rows read from base after the install. + PostImages(CalvinPostImagesSpec), + /// `RETURNING` rows of a resolved timeseries ingest, rendered from what + /// its install stored. + InstalledTimeseries(CalvinInstalledTimeseriesSpec), +} + +/// The engine that stores a row a `RETURNING` reply reads after the install. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub enum CalvinRowEngine { + Document, + Crdt, + Kv, + Vector, + Columnar, + Timeseries, +} + +/// One row a `RETURNING` plan wrote: its client identity and its surrogate. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CalvinReplyRow { + pub identity: String, + pub surrogate: Surrogate, +} + +/// The rows of a `RETURNING` plan read from base after the install. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CalvinPostImagesSpec { + pub spec: ReturningSpec, + pub rls_filters: Vec, + pub collection: String, + pub engine: CalvinRowEngine, + /// Each row the plan wrote, in the order the plan wrote them. + pub rows: Vec, +} + +/// The `RETURNING` projection of a resolved timeseries ingest. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CalvinInstalledTimeseriesSpec { + pub spec: ReturningSpec, + pub rls_filters: Vec, + pub collection: String, + /// The ingest's position among the slice's timeseries ingests. Its + /// install is the install at this position. + pub ordinal: u64, +} + +/// What a stamped redo install of a Calvin slice needs beyond the redo +/// record. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CalvinInstall { + pub epoch: u64, + pub position: u32, + /// The epoch's deterministic instant. The install stamps graph versions + /// at the transaction's ordinal and renders the reply at this instant. + pub epoch_system_ms: i64, + /// The reply the install answers with. + pub reply: CalvinReplySpec, +} + +/// The answer of `MetaOp::CalvinResolve`: the slice's resolved redo record +/// and the reply its stage decided. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CalvinResolved { + /// The zerompk-encoded redo record. + pub redo: Vec, + pub reply: CalvinReplySpec, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::physical_plan::ReturningColumns; + + #[test] + fn a_resolved_answer_round_trips() { + let resolved = CalvinResolved { + redo: vec![1, 2, 3], + reply: CalvinReplySpec::PostImages(CalvinPostImagesSpec { + spec: ReturningSpec { + columns: ReturningColumns::Star, + }, + rls_filters: vec![9], + collection: "orders".into(), + engine: CalvinRowEngine::Kv, + rows: vec![CalvinReplyRow { + identity: "k1".into(), + surrogate: Surrogate::new(4), + }], + }), + }; + let bytes = zerompk::to_msgpack_vec(&resolved).expect("encode"); + let decoded: CalvinResolved = zerompk::from_msgpack(&bytes).expect("decode"); + assert_eq!(decoded, resolved); + } +} diff --git a/nodedb-physical/src/physical_plan/collection.rs b/nodedb-physical/src/physical_plan/collection.rs index 4bd095a99..403ad813c 100644 --- a/nodedb-physical/src/physical_plan/collection.rs +++ b/nodedb-physical/src/physical_plan/collection.rs @@ -190,6 +190,7 @@ mod tests { collections: collections.clone(), sum_targets: Vec::new(), origin: crate::physical_plan::RedoOrigin::Commit, + calvin: None, }); let flush = PhysicalPlan::Meta(MetaOp::CalvinFlush { epoch: 1, diff --git a/nodedb-physical/src/physical_plan/meta.rs b/nodedb-physical/src/physical_plan/meta.rs index 52b842968..54fa96f3e 100644 --- a/nodedb-physical/src/physical_plan/meta.rs +++ b/nodedb-physical/src/physical_plan/meta.rs @@ -583,6 +583,9 @@ pub enum MetaOp { /// (see `calvin_synthetic_txn_id`). Dispatched by the scheduler once the /// global verdict is commit, ahead of `CalvinFlush`, which installs the /// record this op returns. No base engine is touched during resolve. + /// + /// The response payload is a zerompk-encoded [`super::CalvinResolved`]: + /// the redo record and the reply the staged plans decided. CalvinResolve { epoch: u64, position: u32 }, /// Apply one committed transaction's resolved redo record on the core that @@ -598,11 +601,17 @@ pub enum MetaOp { /// version at the record's LSN. `sum_targets` is the materialized-sum /// resolution the transaction's document writes fold into their targets. /// `origin` decides which commit-boundary checks the apply runs. + /// + /// `calvin` is set when the record is a committed Calvin slice. The + /// install then stamps the slice's ordinal on the core clock, consumes + /// the slice's staged state when this core staged it, and answers the + /// reply `calvin.reply` describes. ApplyTransactionRedo { redo: Vec, collections: Vec, sum_targets: Vec, origin: super::RedoOrigin, + calvin: Option, }, /// Install one batch of a RESTORE as part of a Calvin transaction, on diff --git a/nodedb-physical/src/physical_plan/mod.rs b/nodedb-physical/src/physical_plan/mod.rs index 5e1919506..f403546ae 100644 --- a/nodedb-physical/src/physical_plan/mod.rs +++ b/nodedb-physical/src/physical_plan/mod.rs @@ -7,6 +7,7 @@ //! isolated. pub mod array; +pub mod calvin_reply; pub mod cluster_array; pub mod cluster_event; pub mod collection; @@ -37,6 +38,10 @@ pub mod vector; pub mod wire; pub use array::{ArrayBinaryOp, ArrayOp, ArrayReducer}; +pub use calvin_reply::{ + CalvinInstall, CalvinInstalledTimeseriesSpec, CalvinPostImagesSpec, CalvinReplyRow, + CalvinReplySpec, CalvinResolved, CalvinRowEngine, +}; pub use cluster_array::ClusterArrayOp; pub use cluster_event::{ClusterEventOp, MAX_REMOTE_CDC_COMMITTED_OFFSETS}; pub use columnar::{ColumnarInsertIntent, ColumnarOp}; diff --git a/nodedb-raft/src/log.rs b/nodedb-raft/src/log.rs index a9ebb232c..a2ecd0279 100644 --- a/nodedb-raft/src/log.rs +++ b/nodedb-raft/src/log.rs @@ -4,6 +4,11 @@ use crate::error::{RaftError, Result}; use crate::message::LogEntry; use crate::storage::LogStorage; +/// Most entry data one AppendEntries request carries. Half the 64 MiB RPC +/// frame limit, so a batch and its envelope fit one frame. A single entry +/// larger than this ships alone, and the host bounds an entry's size. +pub const MAX_APPEND_BATCH_BYTES: usize = 32 * 1024 * 1024; + /// In-memory Raft log backed by a pluggable `LogStorage`. /// /// The log is 1-indexed. Index 0 is a sentinel (term 0, empty data). @@ -78,6 +83,23 @@ impl RaftLog { self.entries.get(offset) } + /// Entries from `lo` up to `hi` inclusive whose data sums to at most + /// `max_bytes`. The first entry is always included, so an entry larger + /// than `max_bytes` still ships, alone. + pub fn entries_within(&self, lo: u64, hi: u64, max_bytes: usize) -> Result<&[LogEntry]> { + let range = self.entries_range(lo, hi)?; + let mut total = 0usize; + let mut end = 0usize; + for entry in range { + total = total.saturating_add(entry.data.len()); + if end > 0 && total > max_bytes { + break; + } + end += 1; + } + Ok(&range[..end]) + } + /// Get entries in range [lo, hi] inclusive. pub fn entries_range(&self, lo: u64, hi: u64) -> Result<&[LogEntry]> { if lo <= self.snapshot_index { @@ -265,6 +287,24 @@ mod tests { assert_eq!(range[2].index, 4); } + #[test] + fn entries_within_stops_at_the_byte_budget_and_keeps_one_entry() { + let mut log = RaftLog::new(MemStorage::new()); + for i in 1..=4 { + log.append(LogEntry { + term: 1, + index: i, + data: vec![0; 10], + }) + .unwrap(); + } + assert_eq!(log.entries_within(1, 4, 25).unwrap().len(), 2); + assert_eq!(log.entries_within(1, 4, 40).unwrap().len(), 4); + // An entry past the budget ships alone. + assert_eq!(log.entries_within(2, 4, 5).unwrap().len(), 1); + assert!(log.entries_within(5, 4, 5).unwrap().is_empty()); + } + #[test] fn conflict_detection() { let mut log = RaftLog::new(MemStorage::new()); diff --git a/nodedb-raft/src/node/internal.rs b/nodedb-raft/src/node/internal.rs index 6c74e3655..defb9cb73 100644 --- a/nodedb-raft/src/node/internal.rs +++ b/nodedb-raft/src/node/internal.rs @@ -256,7 +256,11 @@ impl RaftNode { }; let entries = if next_index <= self.log.last_index() { - match self.log.entries_range(next_index, self.log.last_index()) { + match self.log.entries_within( + next_index, + self.log.last_index(), + crate::log::MAX_APPEND_BATCH_BYTES, + ) { Ok(slice) => slice.to_vec(), Err(RaftError::LogCompacted { .. }) => { debug!( @@ -346,7 +350,11 @@ impl RaftNode { }; let entries = if next_index <= self.log.last_index() { - match self.log.entries_range(next_index, self.log.last_index()) { + match self.log.entries_within( + next_index, + self.log.last_index(), + crate::log::MAX_APPEND_BATCH_BYTES, + ) { Ok(slice) => slice.to_vec(), Err(crate::error::RaftError::LogCompacted { .. }) => { debug!( diff --git a/nodedb-test-support/src/tx_commit.rs b/nodedb-test-support/src/tx_commit.rs index 62bc8fade..31cbc7987 100644 --- a/nodedb-test-support/src/tx_commit.rs +++ b/nodedb-test-support/src/tx_commit.rs @@ -84,6 +84,7 @@ pub fn commit_plans( collections: written_collections(&plans), sum_targets: redo_sum_targets(&plans), origin: nodedb_physical::physical_plan::RedoOrigin::Commit, + calvin: None, })); install.wal_lsn = Some(Lsn::new(lsn)); send_request(core, tx, rx, install) diff --git a/nodedb-types/src/config/tuning/calvin.rs b/nodedb-types/src/config/tuning/calvin.rs index cc98d759f..107570077 100644 --- a/nodedb-types/src/config/tuning/calvin.rs +++ b/nodedb-types/src/config/tuning/calvin.rs @@ -37,6 +37,17 @@ fn default_catch_up_window() -> u64 { 512 } +fn default_max_redo_entry_bytes() -> usize { + // Well below the 64 MiB RPC frame limit, so one entry and the frame + // around it always fit. + 8 * 1024 * 1024 +} + +fn default_max_open_redo_bytes() -> u64 { + // Sixty-four streams of the default entry size. + 512 * 1024 * 1024 +} + /// Tuning knobs for the Calvin scheduler that runs per hosted vShard. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CalvinTuning { @@ -69,6 +80,17 @@ pub struct CalvinTuning { /// Most sequencer log entries one catch-up drain reads and replays. #[serde(default = "default_catch_up_window")] pub catch_up_window: u64, + + /// Largest encoded redo entry a proposer puts in a data-group log. A + /// committed redo past it travels as chunk entries and a final entry. + #[serde(default = "default_max_redo_entry_bytes")] + pub max_redo_entry_bytes: usize, + + /// Most bytes of chunked redo streams a node holds open. A group leader + /// refuses a new stream that would pass it on the leader's node. A node + /// holds at most this much per data group it hosts. + #[serde(default = "default_max_open_redo_bytes")] + pub max_open_redo_bytes: u64, } impl Default for CalvinTuning { @@ -80,6 +102,8 @@ impl Default for CalvinTuning { verdict_stall_warn_ms: default_verdict_stall_warn_ms(), max_inflight_backlog: default_max_inflight_backlog(), catch_up_window: default_catch_up_window(), + max_redo_entry_bytes: default_max_redo_entry_bytes(), + max_open_redo_bytes: default_max_open_redo_bytes(), } } } @@ -97,6 +121,8 @@ mod tests { assert_eq!(t.verdict_stall_warn_ms, 5_000); assert_eq!(t.max_inflight_backlog, 1_024); assert_eq!(t.catch_up_window, 512); + assert_eq!(t.max_redo_entry_bytes, 8 * 1024 * 1024); + assert_eq!(t.max_open_redo_bytes, 512 * 1024 * 1024); } #[test] diff --git a/nodedb-wal/src/record/types.rs b/nodedb-wal/src/record/types.rs index 19a16c25f..9fb4907e3 100644 --- a/nodedb-wal/src/record/types.rs +++ b/nodedb-wal/src/record/types.rs @@ -423,6 +423,16 @@ pub enum RecordType { /// /// Required: skipping it leaves every truncated edge live after replay. GraphEdgeCut = 68 | 0x8000, + + /// One piece of a chunk of a chunked transaction redo stream, or the + /// close of such a stream. Payload: a `RedoChunkRecord`. A chunk past the + /// WAL record limit spans several pieces, and only its last piece carries + /// the chunk entry's apply key. Boot rebuilds every open stream from these + /// records. Never replayed into any engine. + /// + /// Required: skipping it loses the bytes an open stream holds, and the + /// stream's final entry then refuses on this replica alone. + RedoChunk = 69 | 0x8000, } impl RecordType { @@ -484,6 +494,7 @@ impl RecordType { x if x == 66 | 0x8000 => Some(Self::SnapshotInstalled), x if x == 67 | 0x8000 => Some(Self::WriteGroup), x if x == 68 | 0x8000 => Some(Self::GraphEdgeCut), + x if x == 69 | 0x8000 => Some(Self::RedoChunk), _ => None, } } @@ -521,6 +532,7 @@ mod tests { // Without the flag an older reader skips the abort marker and replays // the refused write it names — the exact bug it exists to prevent. assert!(RecordType::is_required(RecordType::WriteAborted as u32)); + assert!(RecordType::is_required(RecordType::RedoChunk as u32)); } #[test] @@ -576,6 +588,7 @@ mod tests { RecordType::SnapshotInstalled, RecordType::WriteGroup, RecordType::GraphEdgeCut, + RecordType::RedoChunk, ] { assert_eq!(RecordType::from_raw(ty as u32), Some(ty)); } diff --git a/nodedb/src/bootstrap/calvin_catch_up.rs b/nodedb/src/bootstrap/calvin_catch_up.rs index 7419b0611..2ebe00f5f 100644 --- a/nodedb/src/bootstrap/calvin_catch_up.rs +++ b/nodedb/src/bootstrap/calvin_catch_up.rs @@ -69,7 +69,7 @@ fn describe_lagging(lagging: &[u32]) -> String { /// Hold startup until every Calvin scheduler on this node reaches its rebuild /// target. pub async fn await_calvin_catch_up(shared: &Arc) -> anyhow::Result<()> { - let registry = shared.authorization_fence.calvin_mirrors().caught_up(); + let registry = &shared.calvin.caught_up; let deadline = Instant::now() + DATA_GROUP_RECOVERY_TIMEOUT; loop { diff --git a/nodedb/src/config/server/config.rs b/nodedb/src/config/server/config.rs index 7c24ecf69..265e1b607 100644 --- a/nodedb/src/config/server/config.rs +++ b/nodedb/src/config/server/config.rs @@ -473,6 +473,55 @@ mod tests { assert_eq!(cfg.tuning.calvin.channel_capacity, 512); } + #[test] + fn from_file_rejects_a_redo_entry_size_past_the_rpc_frame() { + let path = write_temp_config( + "nodedb-domain-redo-entry-large.toml", + "[tuning.calvin]\nmax_redo_entry_bytes = 67108864\n", + ); + let err = ServerConfig::from_file(&path).unwrap_err(); + std::fs::remove_file(&path).ok(); + let msg = err.to_string(); + assert!(msg.contains("tuning.calvin.max_redo_entry_bytes"), "{msg}"); + assert!(msg.contains("RPC frame limit"), "{msg}"); + } + + #[test] + fn from_file_rejects_a_zero_redo_entry_size() { + let path = write_temp_config( + "nodedb-domain-redo-entry-zero.toml", + "[tuning.calvin]\nmax_redo_entry_bytes = 0\n", + ); + let err = ServerConfig::from_file(&path).unwrap_err(); + std::fs::remove_file(&path).ok(); + let msg = err.to_string(); + assert!(msg.contains("tuning.calvin.max_redo_entry_bytes"), "{msg}"); + } + + #[test] + fn from_file_rejects_an_open_redo_cap_below_one_entry() { + let path = write_temp_config( + "nodedb-domain-redo-open-cap.toml", + "[tuning.calvin]\nmax_redo_entry_bytes = 1048576\nmax_open_redo_bytes = 65536\n", + ); + let err = ServerConfig::from_file(&path).unwrap_err(); + std::fs::remove_file(&path).ok(); + let msg = err.to_string(); + assert!(msg.contains("tuning.calvin.max_open_redo_bytes"), "{msg}"); + } + + #[test] + fn from_file_reads_the_redo_sizes() { + let path = write_temp_config( + "nodedb-domain-redo-ok.toml", + "[tuning.calvin]\nmax_redo_entry_bytes = 65536\nmax_open_redo_bytes = 1048576\n", + ); + let cfg = ServerConfig::from_file(&path).expect("valid redo sizes"); + std::fs::remove_file(&path).ok(); + assert_eq!(cfg.tuning.calvin.max_redo_entry_bytes, 65_536); + assert_eq!(cfg.tuning.calvin.max_open_redo_bytes, 1_048_576); + } + /// Every shipped default satisfies every bound the gate enforces. #[test] fn the_compiled_defaults_are_in_domain() { diff --git a/nodedb/src/config/server/domain.rs b/nodedb/src/config/server/domain.rs index bad851ab9..cba06ab96 100644 --- a/nodedb/src/config/server/domain.rs +++ b/nodedb/src/config/server/domain.rs @@ -22,6 +22,18 @@ pub(super) const MIN_SCOPE_EXPIRY_SECS: u64 = 10; /// ticks at a quarter of it, and a zero-length tick panics the timer. pub(super) const MIN_CALVIN_VERDICT_STALL_WARN_MS: u64 = 4; +/// Smallest redo entry size. A chunk entry carries its stream header beside +/// its bytes, so a smaller entry leaves too little room for the bytes. +pub(super) const MIN_REDO_ENTRY_BYTES: usize = 64 * 1024; + +/// Room a redo entry leaves in one RPC frame for the frame header, the +/// AppendEntries envelope, and the entry's own routing fields. +pub(super) const REDO_ENTRY_ENVELOPE_BYTES: usize = 1024 * 1024; + +/// Largest redo entry size: one entry and its envelope fit one RPC frame. +pub(super) const MAX_REDO_ENTRY_BYTES: usize = + nodedb_cluster::rpc_codec::MAX_RPC_PAYLOAD_SIZE as usize - REDO_ENTRY_ENVELOPE_BYTES; + /// Rejects an endpoint that carries no `http://` or `https://` host. pub(super) fn otlp_endpoint_has_host(raw: &str) -> bool { raw.strip_prefix("http://") @@ -160,5 +172,30 @@ fn validate_calvin(config: &ServerConfig) -> crate::Result<()> { calvin.max_inflight_backlog, "tuning.calvin.max_inflight_backlog", )?; - positive_u64(calvin.catch_up_window, "tuning.calvin.catch_up_window") + positive_u64(calvin.catch_up_window, "tuning.calvin.catch_up_window")?; + validate_redo_sizes(calvin) +} + +/// Checks the redo entry size against one RPC frame, and the open-stream +/// cap against one entry. +fn validate_redo_sizes(calvin: &nodedb_types::config::tuning::CalvinTuning) -> crate::Result<()> { + let entry = calvin.max_redo_entry_bytes; + if !(MIN_REDO_ENTRY_BYTES..=MAX_REDO_ENTRY_BYTES).contains(&entry) { + return Err(reject( + "tuning.calvin.max_redo_entry_bytes", + entry, + &format!( + "a size of at least {MIN_REDO_ENTRY_BYTES} bytes and at most \ + {MAX_REDO_ENTRY_BYTES} bytes, the RPC frame limit less its envelope" + ), + )); + } + if calvin.max_open_redo_bytes < entry as u64 { + return Err(reject( + "tuning.calvin.max_open_redo_bytes", + calvin.max_open_redo_bytes, + "a size of at least tuning.calvin.max_redo_entry_bytes", + )); + } + Ok(()) } diff --git a/nodedb/src/control/backup/restore/sections.rs b/nodedb/src/control/backup/restore/sections.rs index fc83afab6..995c09e1c 100644 --- a/nodedb/src/control/backup/restore/sections.rs +++ b/nodedb/src/control/backup/restore/sections.rs @@ -108,6 +108,8 @@ pub(in crate::control::backup) fn append_snapshot( // A backup carries no Calvin cut: the restore re-applies its rows // as new writes on the destination. group_calvin: _, + // A backup carries no redo stream: its rows restore as new writes. + group_redo_streams: _, // A backup carries visible edge versions only. The restore re-applies // them at its own ordinal, so the source's cuts, hidden versions and // applied ordinals never reach the destination. diff --git a/nodedb/src/control/checkpoint_manager.rs b/nodedb/src/control/checkpoint_manager.rs index 18a2f4652..8a71cc7f7 100644 --- a/nodedb/src/control/checkpoint_manager.rs +++ b/nodedb/src/control/checkpoint_manager.rs @@ -173,16 +173,17 @@ pub struct CheckpointCycleInputs<'a> { pub archiver: Option<&'a mut WalArchiver>, /// When present, the tombstone set is GC'd to the new truncation point. pub catalog: Option<&'a crate::control::security::catalog::SystemCatalog>, - /// The applied state of every Calvin scheduler on this node. Saved in + /// The Calvin applied ledger of every vShard on this node. Saved in /// `catalog` before truncation deletes the applied markers it came from. - pub calvin_mirrors: Option<&'a crate::control::cluster::calvin::scheduler::AppliedMirrors>, + pub calvin_ledgers: + Option<&'a crate::control::cluster::calvin::scheduler::CalvinAppliedLedgers>, } -/// The applied state of every mirror, as the catalog stores it. +/// The applied state of every ledger, as the catalog stores it. fn calvin_applied_states( - mirrors: &crate::control::cluster::calvin::scheduler::AppliedMirrors, + ledgers: &crate::control::cluster::calvin::scheduler::CalvinAppliedLedgers, ) -> Vec { - mirrors.snapshot_all() + ledgers.snapshot_all() } /// Save `states` in `catalog`. @@ -215,7 +216,7 @@ pub async fn run_checkpoint_cycle(inputs: CheckpointCycleInputs<'_>) -> Option) -> Option, + /// Positions an install holds and has not finished. + claims: BTreeSet<(u64, u32)>, +} + +impl LedgerState { + fn below_watermark(&self, epoch: u64) -> bool { + self.fully_applied_epoch != NOT_YET_APPLIED_EPOCH && epoch <= self.fully_applied_epoch + } + + fn is_applied(&self, epoch: u64, position: u32) -> bool { + self.below_watermark(epoch) || self.tail.contains(&(epoch, position)) + } + + fn insert_applied(&mut self, epoch: u64, position: u32) { + if !self.below_watermark(epoch) { + self.tail.insert((epoch, position)); + } + } +} + +/// Applied positions of one vShard on this node. +#[derive(Debug)] +pub struct CalvinAppliedLedger { + state: Mutex, +} + +impl CalvinAppliedLedger { + /// A ledger holding the watermark `fully_applied_epoch` and the applied + /// positions `tail` above it, with no claim. + pub fn new(fully_applied_epoch: u64, tail: BTreeSet<(u64, u32)>) -> Self { + Self { + state: Mutex::new(LedgerState { + fully_applied_epoch, + tail, + claims: BTreeSet::new(), + }), + } + } + + /// Reserve `(epoch, position)` for one install. Refuses an applied + /// position and a position another install holds. + pub fn claim(&self, epoch: u64, position: u32) -> Result<(), ClaimRefusal> { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + if state.is_applied(epoch, position) { + return Err(ClaimRefusal::Applied); + } + if !state.claims.insert((epoch, position)) { + return Err(ClaimRefusal::Claimed); + } + Ok(()) + } + + /// Drop the claim on `(epoch, position)`: its install did not apply, so a + /// later copy can install it. + pub fn release_claim(&self, epoch: u64, position: u32) { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + state.claims.remove(&(epoch, position)); + } + + /// Record that the install of `(epoch, position)` is durable. Drops its + /// claim. + pub fn mark_applied(&self, epoch: u64, position: u32) { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + state.claims.remove(&(epoch, position)); + state.insert_applied(epoch, position); + } + + /// Record that `(epoch, position)` finished with no install on this + /// vShard: an abort, or a slice that writes nothing. A claim on it stays + /// until its holder finishes. + pub fn mark_terminal(&self, epoch: u64, position: u32) { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + state.insert_applied(epoch, position); + } + + /// Record that every position of every epoch at or below `watermark` + /// applied. A lower watermark changes nothing. + pub fn fold(&self, watermark: u64) { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + if state.below_watermark(watermark) { + return; + } + state.fully_applied_epoch = watermark; + let above = (watermark.saturating_add(1), 0); + state.tail = state.tail.split_off(&above); + state.claims = state.claims.split_off(&above); + } + + /// The fully-applied watermark and the applied positions above it. + /// Claims are not applied positions, so they are absent. + pub fn snapshot(&self) -> (u64, BTreeSet<(u64, u32)>) { + let state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + (state.fully_applied_epoch, state.tail.clone()) + } + + /// Whether this replica applied `(epoch, position)`. + pub fn is_applied(&self, epoch: u64, position: u32) -> bool { + let state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + state.is_applied(epoch, position) + } + + /// The highest epoch with an applied position, or + /// [`NOT_YET_APPLIED_EPOCH`] when none is applied. + pub fn max_applied_epoch(&self) -> u64 { + let state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + match state.tail.last() { + Some((epoch, _)) => *epoch, + None => state.fully_applied_epoch, + } + } +} + +/// The applied ledger of every vShard this node holds Calvin state for. +#[derive(Debug, Default)] +pub struct CalvinAppliedLedgers { + by_vshard: Mutex>>, +} + +impl CalvinAppliedLedgers { + /// Replace the ledger of `vshard_id` with one holding + /// `fully_applied_epoch` and `tail`. A holder of the old ledger keeps the + /// old one. + pub fn install( + &self, + vshard_id: u32, + fully_applied_epoch: u64, + tail: BTreeSet<(u64, u32)>, + ) -> Arc { + let ledger = Arc::new(CalvinAppliedLedger::new(fully_applied_epoch, tail)); + self.by_vshard + .lock() + .unwrap_or_else(|p| p.into_inner()) + .insert(vshard_id, Arc::clone(&ledger)); + ledger + } + + /// The ledger of `vshard_id`. A vShard with no ledger has no Calvin + /// history on this node, so it gets an empty one. + pub fn get_or_create(&self, vshard_id: u32) -> Arc { + let mut by_vshard = self.by_vshard.lock().unwrap_or_else(|p| p.into_inner()); + Arc::clone(by_vshard.entry(vshard_id).or_insert_with(|| { + Arc::new(CalvinAppliedLedger::new( + NOT_YET_APPLIED_EPOCH, + BTreeSet::new(), + )) + })) + } + + /// The ledger of `vshard_id`, when this node holds one. + pub fn get(&self, vshard_id: u32) -> Option> { + self.by_vshard + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&vshard_id) + .cloned() + } + + /// Drop the ledger of `vshard_id`: this node left the vShard's group and + /// holds none of its applied positions. + pub fn remove(&self, vshard_id: u32) { + self.by_vshard + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(&vshard_id); + } + + /// Every ledger's applied state, in the shape the catalog stores. + pub fn snapshot_all(&self) -> Vec { + let ledgers: Vec<(u32, Arc)> = self + .by_vshard + .lock() + .unwrap_or_else(|p| p.into_inner()) + .iter() + .map(|(vshard_id, ledger)| (*vshard_id, Arc::clone(ledger))) + .collect(); + ledgers + .into_iter() + .map(|(vshard_id, ledger)| { + let (fully_applied_epoch, tail) = ledger.snapshot(); + StoredCalvinApplied { + vshard_id, + fully_applied_epoch, + tail, + } + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn empty() -> CalvinAppliedLedger { + CalvinAppliedLedger::new(NOT_YET_APPLIED_EPOCH, BTreeSet::new()) + } + + #[test] + fn a_ledger_answers_like_the_gate() { + let ledger = empty(); + assert!(!ledger.is_applied(0, 0)); + ledger.mark_applied(3, 1); + assert!(ledger.is_applied(3, 1)); + assert!(!ledger.is_applied(3, 0)); + ledger.fold(3); + assert!(ledger.is_applied(3, 0)); + assert!(ledger.is_applied(2, 9)); + assert!(!ledger.is_applied(4, 0)); + // A mark at or below the watermark changes nothing. + ledger.mark_terminal(1, 0); + assert!(ledger.is_applied(1, 0)); + // A lower watermark never moves it back. + ledger.fold(1); + assert!(ledger.is_applied(3, 0)); + assert_eq!(ledger.max_applied_epoch(), 3); + } + + #[test] + fn a_claim_refuses_a_claimed_or_applied_position() { + let ledger = empty(); + assert_eq!(ledger.claim(2, 0), Ok(())); + assert_eq!(ledger.claim(2, 0), Err(ClaimRefusal::Claimed)); + ledger.mark_applied(2, 0); + assert_eq!(ledger.claim(2, 0), Err(ClaimRefusal::Applied)); + ledger.mark_terminal(2, 1); + assert_eq!(ledger.claim(2, 1), Err(ClaimRefusal::Applied)); + ledger.fold(5); + assert_eq!(ledger.claim(4, 7), Err(ClaimRefusal::Applied)); + assert_eq!(ledger.claim(6, 0), Ok(())); + } + + #[test] + fn a_released_claim_can_be_claimed_again() { + let ledger = empty(); + assert_eq!(ledger.claim(1, 3), Ok(())); + ledger.release_claim(1, 3); + assert!(!ledger.is_applied(1, 3)); + assert_eq!(ledger.claim(1, 3), Ok(())); + } + + #[test] + fn snapshot_names_only_applied_positions() { + let ledger = empty(); + ledger.claim(4, 0).expect("claim"); + ledger.claim(4, 1).expect("claim"); + ledger.mark_applied(4, 1); + ledger.mark_terminal(5, 2); + let (watermark, tail) = ledger.snapshot(); + assert_eq!(watermark, NOT_YET_APPLIED_EPOCH); + assert_eq!(tail, [(4, 1), (5, 2)].into_iter().collect()); + } + + #[test] + fn an_empty_ledger_has_no_applied_epoch() { + assert_eq!(empty().max_applied_epoch(), NOT_YET_APPLIED_EPOCH); + let folded = CalvinAppliedLedger::new(6, BTreeSet::new()); + assert_eq!(folded.max_applied_epoch(), 6); + } + + #[test] + fn an_installed_ledger_replaces_the_previous_one() { + let ledgers = CalvinAppliedLedgers::default(); + let first = ledgers.install(7, NOT_YET_APPLIED_EPOCH, BTreeSet::new()); + first.mark_applied(1, 0); + let second = ledgers.install(7, 4, BTreeSet::new()); + let current = ledgers.get(7).expect("ledger"); + assert!(Arc::ptr_eq(¤t, &second)); + assert!(current.is_applied(4, 0)); + assert!(ledgers.get(8).is_none()); + } + + #[test] + fn get_or_create_keeps_an_existing_ledger() { + let ledgers = CalvinAppliedLedgers::default(); + let recovered = ledgers.install(3, NOT_YET_APPLIED_EPOCH, [(2, 0)].into_iter().collect()); + let used = ledgers.get_or_create(3); + assert!(Arc::ptr_eq(&recovered, &used)); + assert!(used.is_applied(2, 0)); + let fresh = ledgers.get_or_create(9); + assert!(!fresh.is_applied(0, 0)); + assert_eq!(ledgers.snapshot_all().len(), 2); + ledgers.remove(9); + assert!(ledgers.get(9).is_none()); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/applied_mirror.rs b/nodedb/src/control/cluster/calvin/scheduler/applied_mirror.rs deleted file mode 100644 index 3ae5ed170..000000000 --- a/nodedb/src/control/cluster/calvin/scheduler/applied_mirror.rs +++ /dev/null @@ -1,195 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! A shared view of which Calvin positions this node's scheduler for one -//! vShard applied. -//! -//! The scheduler owns its [`super::AppliedGate`] and mutates it on its own -//! task. Other Control-Plane code needs one question answered: did this -//! node's replica of the vShard apply `(epoch, position)`? The scheduler -//! mirrors each applied position and each watermark advance here, so the -//! answer needs no message to the scheduler task. -//! -//! The mirror keeps the same shape as the gate: a fully-applied watermark and -//! the applied positions above it. The tail is pruned as the watermark -//! advances, so it stays as small as the gate's. -//! -//! [`AppliedMirrors`] also holds each running scheduler's caught-up state, -//! which startup reads before it opens the client gateway. - -use std::collections::{BTreeSet, HashMap}; -use std::sync::{Arc, Mutex}; - -use super::caught_up::CaughtUpRegistry; -use super::recovery::NOT_YET_APPLIED_EPOCH; -use crate::control::security::catalog::calvin_applied::StoredCalvinApplied; - -#[derive(Debug)] -struct MirrorState { - /// Every position of every epoch at or below this is applied. - /// [`NOT_YET_APPLIED_EPOCH`] means none is. - fully_applied_epoch: u64, - /// Applied positions of epochs above the watermark. - tail: BTreeSet<(u64, u32)>, -} - -/// Applied positions of one vShard's scheduler on this node. -#[derive(Debug)] -pub struct AppliedMirror { - state: Mutex, -} - -impl AppliedMirror { - /// A mirror seeded from the scheduler's recovery scan. - pub fn new(fully_applied_epoch: u64, tail: BTreeSet<(u64, u32)>) -> Self { - Self { - state: Mutex::new(MirrorState { - fully_applied_epoch, - tail, - }), - } - } - - /// Record that `(epoch, position)` applied. - pub fn mark(&self, epoch: u64, position: u32) { - let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); - if state.fully_applied_epoch != NOT_YET_APPLIED_EPOCH && epoch <= state.fully_applied_epoch - { - return; - } - state.tail.insert((epoch, position)); - } - - /// Record that every position of every epoch at or below `watermark` - /// applied. - pub fn fold(&self, watermark: u64) { - let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); - if state.fully_applied_epoch != NOT_YET_APPLIED_EPOCH - && watermark <= state.fully_applied_epoch - { - return; - } - state.fully_applied_epoch = watermark; - state.tail = state.tail.split_off(&(watermark.saturating_add(1), 0)); - } - - /// The mirror's state: the fully-applied watermark and the applied - /// positions above it. - pub fn snapshot(&self) -> (u64, BTreeSet<(u64, u32)>) { - let state = self.state.lock().unwrap_or_else(|p| p.into_inner()); - (state.fully_applied_epoch, state.tail.clone()) - } - - /// Whether this node's replica applied `(epoch, position)`. - pub fn is_applied(&self, epoch: u64, position: u32) -> bool { - let state = self.state.lock().unwrap_or_else(|p| p.into_inner()); - (state.fully_applied_epoch != NOT_YET_APPLIED_EPOCH && epoch <= state.fully_applied_epoch) - || state.tail.contains(&(epoch, position)) - } -} - -/// The applied mirror of every vShard scheduler on this node. -#[derive(Debug, Default)] -pub struct AppliedMirrors { - by_vshard: Mutex>>, - /// Whether each running scheduler reached its rebuild target. - caught_up: CaughtUpRegistry, -} - -impl AppliedMirrors { - /// Caught-up state of every running scheduler on this node. - pub fn caught_up(&self) -> &CaughtUpRegistry { - &self.caught_up - } - - /// Register the mirror of a scheduler starting for `vshard_id`. A - /// restarted scheduler replaces its predecessor's mirror. - pub fn register( - &self, - vshard_id: u32, - fully_applied_epoch: u64, - tail: &BTreeSet<(u64, u32)>, - ) -> Arc { - let mirror = Arc::new(AppliedMirror::new(fully_applied_epoch, tail.clone())); - self.by_vshard - .lock() - .unwrap_or_else(|p| p.into_inner()) - .insert(vshard_id, Arc::clone(&mirror)); - mirror - } - - /// Every registered mirror's state, in the shape the catalog stores. - pub fn snapshot_all(&self) -> Vec { - let mirrors: Vec<(u32, Arc)> = self - .by_vshard - .lock() - .unwrap_or_else(|p| p.into_inner()) - .iter() - .map(|(vshard_id, mirror)| (*vshard_id, Arc::clone(mirror))) - .collect(); - mirrors - .into_iter() - .map(|(vshard_id, mirror)| { - let (fully_applied_epoch, tail) = mirror.snapshot(); - StoredCalvinApplied { - vshard_id, - fully_applied_epoch, - tail, - } - }) - .collect() - } - - /// Drop the mirror of `vshard_id`: this node left the vShard's group and - /// holds none of its applied positions. - pub fn remove(&self, vshard_id: u32) { - self.by_vshard - .lock() - .unwrap_or_else(|p| p.into_inner()) - .remove(&vshard_id); - } - - /// The mirror of `vshard_id`, when this node runs its scheduler. - pub fn get(&self, vshard_id: u32) -> Option> { - self.by_vshard - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get(&vshard_id) - .cloned() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn a_mirror_answers_like_the_gate() { - let mirror = AppliedMirror::new(NOT_YET_APPLIED_EPOCH, BTreeSet::new()); - assert!(!mirror.is_applied(0, 0)); - mirror.mark(3, 1); - assert!(mirror.is_applied(3, 1)); - assert!(!mirror.is_applied(3, 0)); - mirror.fold(3); - assert!(mirror.is_applied(3, 0)); - assert!(mirror.is_applied(2, 9)); - assert!(!mirror.is_applied(4, 0)); - // A mark at or below the watermark changes nothing. - mirror.mark(1, 0); - assert!(mirror.is_applied(1, 0)); - // A lower watermark never moves it back. - mirror.fold(1); - assert!(mirror.is_applied(3, 0)); - } - - #[test] - fn a_restarted_scheduler_replaces_its_mirror() { - let mirrors = AppliedMirrors::default(); - let first = mirrors.register(7, NOT_YET_APPLIED_EPOCH, &BTreeSet::new()); - first.mark(1, 0); - let second = mirrors.register(7, 4, &BTreeSet::new()); - let current = mirrors.get(7).expect("mirror"); - assert!(Arc::ptr_eq(¤t, &second)); - assert!(current.is_applied(4, 0)); - assert!(mirrors.get(8).is_none()); - } -} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_redo.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_redo.rs index 8a21b2fd6..555471d6c 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_redo.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_redo.rs @@ -49,7 +49,19 @@ impl Scheduler { return; } - let mut redo = match RedoRecord::from_bytes(response.payload.as_bytes()) { + // The answer carries the slice's staged reply beside the record. The + // flush renders the reply from the core's staged entry. + let resolved = zerompk::from_msgpack::( + response.payload.as_bytes(), + ); + let decoded = match resolved { + Ok(resolved) => RedoRecord::from_bytes(&resolved.redo), + Err(e) => Err(crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("CalvinResolve answer: {e}"), + }), + }; + let mut redo = match decoded { Ok(r) => r, Err(e) => { self.halt_apply( diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs index e784f0f8c..ecc90c5f8 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs @@ -277,7 +277,12 @@ mod tests { row_changes: Vec::new(), }; let mut response = staged_response(Status::Ok, None); - response.payload = Payload::from_vec(redo.to_bytes().expect("encode empty redo record")); + let resolved = nodedb_physical::physical_plan::CalvinResolved { + redo: redo.to_bytes().expect("encode empty redo record"), + reply: nodedb_physical::physical_plan::CalvinReplySpec::Count(Vec::new()), + }; + response.payload = + Payload::from_vec(zerompk::to_msgpack_vec(&resolved).expect("encode resolved answer")); response } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs index 54e31ae6a..fd236b620 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs @@ -287,11 +287,16 @@ impl Scheduler { ); return; }; - // The flush applied the txn's redo record. - if let Some(records) = pending.redo_records { - records.settle(); - } - self.release_and_mark_applied(txn_id, pending.lock_owner); + // The flush applied the txn's redo record. A txn with no record + // finished without an install on this vShard. + let installed = match pending.redo_records { + Some(records) => { + records.settle(); + true + } + None => false, + }; + self.release_and_mark_applied(txn_id, pending.lock_owner, installed); } /// Complete a txn that never entered `pending`, releasing the locks held @@ -302,12 +307,13 @@ impl Scheduler { txn_id: TxnId, lock_owner: TxnId, ) { - self.release_and_mark_applied(txn_id, lock_owner); + self.release_and_mark_applied(txn_id, lock_owner, false); } /// Release `lock_owner`'s locks, dispatch the promoted waiters, and mark - /// `txn_id`'s position applied. - fn release_and_mark_applied(&mut self, txn_id: TxnId, lock_owner: TxnId) { + /// `txn_id`'s position applied. `installed` says whether a durable redo + /// install finished the position. + fn release_and_mark_applied(&mut self, txn_id: TxnId, lock_owner: TxnId, installed: bool) { // Release this txn's locks. `release` promotes any waiter queued behind // each freed key to holder (moving it pending -> held) and returns the // fully-promoted ids. Those ids are already holders in the table the @@ -326,7 +332,11 @@ impl Scheduler { // so any advertised watermark reflects a FULLY-applied epoch — the value // `BEGIN` needs for a torn-free cross-shard snapshot anchor. let folded = self.applied.mark_applied(txn_id.epoch, txn_id.position); - self.applied_mirror.mark(txn_id.epoch, txn_id.position); + if installed { + self.ledger.mark_applied(txn_id.epoch, txn_id.position); + } else { + self.ledger.mark_terminal(txn_id.epoch, txn_id.position); + } if let Some(watermark) = folded { self.publish_watermark(watermark); } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs index 35e5148fb..9b2ea0f06 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs @@ -100,8 +100,8 @@ pub struct Scheduler { /// counter so a multi-position epoch is never marked applied on the strength /// of its first completing position. pub(in crate::control::cluster::calvin::scheduler::driver::core) applied: AppliedGate, - /// Rebuild target epoch (highest applied epoch from the initial recovery - /// scan). + /// Rebuild target epoch: the highest applied epoch of the ledger when + /// the scheduler started. pub(in crate::control::cluster::calvin::scheduler::driver::core) rebuild_target_epoch: u64, /// Highest replicated epoch observed across all scheduler inputs so far. /// Advances monotonically as `process_scheduler_input` sees new inputs; the @@ -113,9 +113,10 @@ pub struct Scheduler { /// set and the markers not yet reported. pub(in crate::control::cluster::calvin::scheduler::driver::core) cut_floors: crate::control::cluster::calvin::scheduler::cut_floor::CutFloors, - /// Shared mirror of `applied`, read by authorization coverage. - pub(in crate::control::cluster::calvin::scheduler::driver::core) applied_mirror: - Arc, + /// This vShard's applied ledger: the gate's seed, and the shared record + /// of each position this scheduler finishes. + pub(in crate::control::cluster::calvin::scheduler::driver::core) ledger: + Arc, /// This scheduler's caught-up entry, read by the startup readiness gate. /// Set once the watermark reaches `rebuild_target_epoch`. pub(in crate::control::cluster::calvin::scheduler::driver::core) caught_up: @@ -200,10 +201,11 @@ pub struct SchedulerParams { /// Shared sequencer state machine, source of the per-vShard catch-up index /// the drain replays from. Same `Arc` the Raft apply loop drives. pub sequencer_state_machine: Arc>, - /// Fully-applied watermark seed from the recovery scan. - pub fully_applied_epoch: u64, - /// Applied `(epoch, position)` pairs seed from the recovery scan. - pub applied_tail: std::collections::BTreeSet<(u64, u32)>, + /// This vShard's applied ledger. The applied gate starts from its + /// state. + pub ledger: Arc, + /// The epoch the scheduler must fully apply before it reports caught + /// up. [`NOT_YET_APPLIED_EPOCH`] means nothing to rebuild. pub rebuild_target_epoch: u64, pub config: SchedulerConfig, pub metrics: Arc, @@ -235,8 +237,7 @@ impl Scheduler { multi_raft, sequencer_proposer, sequencer_state_machine, - fully_applied_epoch, - applied_tail, + ledger, rebuild_target_epoch, config, metrics, @@ -252,9 +253,8 @@ impl Scheduler { let completion_cap = config.channel_capacity; let (completion_tx, completion_rx) = mpsc::channel(completion_cap); - let mirrors = shared.authorization_fence.calvin_mirrors(); - let applied_mirror = mirrors.register(vshard_id, fully_applied_epoch, &applied_tail); - let caught_up = mirrors.caught_up().register(vshard_id); + let (fully_applied_epoch, applied_tail) = ledger.snapshot(); + let caught_up = shared.calvin.caught_up.register(vshard_id); // A backup's cut waits on every scheduler this node runs. shared.calvin.cuts.register(vshard_id); @@ -281,7 +281,7 @@ impl Scheduler { read_result_rx, applied: AppliedGate::new(fully_applied_epoch, applied_tail), cut_floors: Default::default(), - applied_mirror, + ledger, caught_up, rebuild_target_epoch, max_input_epoch: 0, @@ -307,12 +307,12 @@ impl Scheduler { /// Whether the scheduler has caught up to the rebuild target epoch. /// - /// `rebuild_target_epoch` is seeded from `AppliedRecovery::max_applied_epoch` - /// (see `recovery.rs`): [`NOT_YET_APPLIED_EPOCH`] means the WAL scan found NO - /// `CalvinApplied` marker for this vShard at all — a greenfield node with no - /// Calvin history — never a real epoch (epoch 0 with markers reports - /// `max_applied_epoch == 0`, distinct from the sentinel). With nothing to - /// rebuild, such a node is trivially caught up. + /// `rebuild_target_epoch` is seeded from the ledger's highest applied + /// epoch (see `CalvinAppliedLedger::max_applied_epoch`): + /// [`NOT_YET_APPLIED_EPOCH`] means the ledger holds NO applied position + /// for this vShard — a greenfield node with no Calvin history — never a + /// real epoch (epoch 0 with markers reports `0`, distinct from the + /// sentinel). With nothing to rebuild, such a node is trivially caught up. /// /// `fully_applied_epoch()` is conservatively seeded to the same sentinel by /// recovery (the watermark only advances once the sequencer's re-fan-out @@ -363,7 +363,7 @@ impl Scheduler { watermark: u64, ) { self.metrics.update_last_applied_epoch(watermark); - self.applied_mirror.fold(watermark); + self.ledger.fold(watermark); self.shared .calvin .last_applied_epoch @@ -447,12 +447,7 @@ mod tests { } fn lagging(scheduler: &Scheduler) -> Vec { - scheduler - .shared - .authorization_fence - .calvin_mirrors() - .caught_up() - .lagging() + scheduler.shared.calvin.caught_up.lagging() } /// A scheduler with a real rebuild target holds the readiness gate until @@ -472,6 +467,27 @@ mod tests { assert!(lagging(&scheduler).is_empty()); } + /// A published watermark folds the vShard's applied ledger, which + /// coverage, checkpoints and snapshot capture read. + #[tokio::test] + async fn a_published_watermark_folds_the_ledger() { + let (mut scheduler, _dir) = build_test_scheduler(4); + let ledger = scheduler + .shared + .calvin + .applied + .get(4) + .expect("the scheduler's ledger"); + assert!(Arc::ptr_eq(&ledger, &scheduler.ledger)); + assert!(!ledger.is_applied(3, 9)); + + scheduler.applied = AppliedGate::new(3, BTreeSet::new()); + scheduler.publish_watermark(3); + + assert!(ledger.is_applied(3, 9)); + assert!(!ledger.is_applied(4, 0)); + } + /// With no Calvin history there is nothing to rebuild, so the scheduler /// reports caught up without a watermark. #[tokio::test] @@ -489,13 +505,6 @@ mod tests { let shared = Arc::clone(&scheduler.shared); assert_eq!(lagging(&scheduler), vec![4]); drop(scheduler); - assert!( - shared - .authorization_fence - .calvin_mirrors() - .caught_up() - .lagging() - .is_empty() - ); + assert!(shared.calvin.caught_up.lagging().is_empty()); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs index 8f6c6673a..ab7e7726b 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs @@ -2,7 +2,7 @@ //! Shared test fixtures for the Calvin scheduler driver's `core` unit tests. -use std::collections::{BTreeSet, HashMap}; +use std::collections::HashMap; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -22,6 +22,7 @@ use crate::bridge::dispatch::{BridgeResponse, CoreChannelDataSide, Dispatcher}; use crate::bridge::envelope::{ Admission, ErrorCode, ExemptReason, Payload, Priority, Request, Response, StageVote, Status, }; +use crate::control::cluster::calvin::scheduler::SchedulerConfig; use crate::control::cluster::calvin::scheduler::driver::barrier::ReadResultEvent; use crate::control::cluster::calvin::scheduler::driver::core::scheduler::{ Scheduler, SchedulerParams, @@ -30,7 +31,6 @@ use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::Cap use crate::control::cluster::calvin::scheduler::driver::types::{CommitState, PendingTxn}; use crate::control::cluster::calvin::scheduler::lock_manager::{LockManager, TxnId}; use crate::control::cluster::calvin::scheduler::metrics::SchedulerMetrics; -use crate::control::cluster::calvin::scheduler::{NOT_YET_APPLIED_EPOCH, SchedulerConfig}; use crate::control::shutdown::ShutdownWatch; use crate::control::state::SharedState; use crate::types::{DatabaseId, Lsn, ReadConsistency, RequestId, VShardId}; @@ -50,6 +50,7 @@ pub(super) fn build_test_scheduler(vshard_id: u32) -> (Scheduler, tempfile::Temp .pop() .expect("one configured core has one data side"); let shared = SharedState::new(dispatcher, wal).unwrap(); + let ledger = shared.calvin.applied.get_or_create(vshard_id); let rt = RoutingTable::uniform(1, &[1], 1); let multi_raft = Arc::new(Mutex::new(MultiRaft::new(1, rt, dir.path().to_path_buf()))); @@ -74,14 +75,13 @@ pub(super) fn build_test_scheduler(vshard_id: u32) -> (Scheduler, tempfile::Temp multi_raft, sequencer_proposer: CapturingProposer::accepting(), sequencer_state_machine, - // A freshly-built scheduler has applied nothing, so its watermark is the + // A freshly-built scheduler has applied nothing, so its ledger holds the // not-yet-applied sentinel (matching `read_applied_recovery` for a clean - // node). Hardcoding `0` here will instead claim epoch 0 is fully applied, + // node). A watermark of `0` will instead claim epoch 0 is fully applied, // making the exactly-once gate (`AppliedGate::is_applied`) short-circuit // every epoch-0 replay before it reaches the lock table — silently // defeating the end-to-end drain tests below. - fully_applied_epoch: NOT_YET_APPLIED_EPOCH, - applied_tail: BTreeSet::new(), + ledger, rebuild_target_epoch: 0, config: SchedulerConfig::default(), metrics: SchedulerMetrics::new(), @@ -109,6 +109,7 @@ pub(super) fn build_test_scheduler_with_data_side( .pop() .expect("one configured core has one data side"); let shared = SharedState::new(dispatcher, wal).unwrap(); + let ledger = shared.calvin.applied.get_or_create(vshard_id); let rt = RoutingTable::uniform(1, &[1], 1); let multi_raft = Arc::new(Mutex::new(MultiRaft::new(1, rt, dir.path().to_path_buf()))); @@ -133,8 +134,7 @@ pub(super) fn build_test_scheduler_with_data_side( multi_raft, sequencer_proposer: CapturingProposer::accepting(), sequencer_state_machine, - fully_applied_epoch: NOT_YET_APPLIED_EPOCH, - applied_tail: BTreeSet::new(), + ledger, rebuild_target_epoch: 0, config: SchedulerConfig::default(), metrics: SchedulerMetrics::new(), diff --git a/nodedb/src/control/cluster/calvin/scheduler/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/mod.rs index be74f239e..0f194a855 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/mod.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 pub mod applied_gate; -pub mod applied_mirror; +pub mod applied_ledger; pub mod caught_up; pub mod cut_floor; pub mod driver; @@ -11,7 +11,7 @@ mod metrics_flow; pub mod recovery; pub use applied_gate::AppliedGate; -pub use applied_mirror::{AppliedMirror, AppliedMirrors}; +pub use applied_ledger::{CalvinAppliedLedger, CalvinAppliedLedgers, ClaimRefusal}; pub use caught_up::{CaughtUpHandle, CaughtUpRegistry}; pub use driver::{ CalvinReadResultProposal, RaftSequencerProposer, ReadResultEvent, Scheduler, SchedulerConfig, @@ -23,5 +23,5 @@ pub use lock::{AcquireOutcome, HotKeyTable, LockKey, LockManager, LockMode, TxnI pub use lock as lock_manager; pub use metrics::SchedulerMetrics; pub use recovery::{ - AppliedRecovery, NOT_YET_APPLIED_EPOCH, read_applied_recovery, recover_applied, + AppliedRecovery, NOT_YET_APPLIED_EPOCH, read_applied_recovery, recover_all_applied, }; diff --git a/nodedb/src/control/cluster/calvin/scheduler/recovery.rs b/nodedb/src/control/cluster/calvin/scheduler/recovery.rs index cac03b1e4..f5fdb076d 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/recovery.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/recovery.rs @@ -7,9 +7,14 @@ //! records carrying a `calvin_stamp` (write-bearing Calvin transactions //! journal the redo record as their applied-marker instead of a standalone //! `CalvinApplied` marker), returning for a given vShard the union of applied -//! `(epoch, position)` pairs together with a fully-applied watermark. The -//! scheduler uses this on startup to seed its exactly-once applied gate -//! (see [`super::applied_gate::AppliedGate`]). +//! `(epoch, position)` pairs together with a fully-applied watermark. +//! +//! Boot runs [`recover_all_applied`] once, before the data-group apply loop +//! starts: one WAL pass yields every vShard's applied state, merged with the +//! state the last checkpoint saved. It fills the node's applied ledgers (see +//! [`super::applied_ledger::CalvinAppliedLedgers`]), and each scheduler seeds +//! its exactly-once applied gate (see [`super::applied_gate::AppliedGate`]) +//! from its vShard's ledger. //! //! Each `CalvinApplied` marker is per `(epoch, position, vShard)` — one per //! independent transaction position — so the scan preserves `position` rather @@ -26,7 +31,7 @@ //! re-fan-out supplies the counts. `max_applied_epoch` is the highest epoch with //! any marker — the rebuild-target cursor, independent of the exact gate. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use nodedb_wal::record::RecordType; use nodedb_wal::{CalvinAppliedPayload, WalRecord}; @@ -53,6 +58,27 @@ pub struct AppliedRecovery { pub max_applied_epoch: u64, } +impl AppliedRecovery { + /// The state of a vShard with no applied marker. The watermark stays + /// conservative: the applied gate advances it once the re-fan-out + /// supplies the per-epoch expected counts. + fn empty() -> Self { + Self { + fully_applied_epoch: NOT_YET_APPLIED_EPOCH, + applied_tail: BTreeSet::new(), + max_applied_epoch: NOT_YET_APPLIED_EPOCH, + } + } + + /// Record one applied `(epoch, position)`. + fn note(&mut self, epoch: u64, position: u32) { + self.applied_tail.insert((epoch, position)); + if self.max_applied_epoch == NOT_YET_APPLIED_EPOCH || epoch > self.max_applied_epoch { + self.max_applied_epoch = epoch; + } + } +} + /// Scan the WAL and collect this vShard's applied `(epoch, position)` markers. /// /// Returns an empty tail and the [`NOT_YET_APPLIED_EPOCH`] sentinel for a @@ -73,34 +99,51 @@ fn scan_applied( vshard_id: u32, group_id: Option, ) -> crate::Result { + let group_of = |vshard: u32| if vshard == vshard_id { group_id } else { None }; + let mut scans = scan_all(wal, &group_of, Some(vshard_id))?; + Ok(scans + .remove(&vshard_id) + .unwrap_or_else(AppliedRecovery::empty)) +} + +/// Every vShard's applied markers in one pass over the WAL. `only` limits +/// the pass to one vShard. `group_of` names a vShard's data group: a +/// snapshot install of the group drops the vShard's markers before it. +fn scan_all( + wal: &WalManager, + group_of: &dyn Fn(u32) -> Option, + only: Option, +) -> crate::Result> { let records = wal.replay()?; - let mut applied_tail = BTreeSet::new(); - let mut max_applied_epoch = NOT_YET_APPLIED_EPOCH; + let mut scans: BTreeMap = BTreeMap::new(); + let note = + |scans: &mut BTreeMap, vshard_id: u32, epoch: u64, position: u32| { + if only.is_none_or(|wanted| wanted == vshard_id) { + scans + .entry(vshard_id) + .or_insert_with(AppliedRecovery::empty) + .note(epoch, position); + } + }; for record in &records { match record_type_of(record) { Some(RecordType::SnapshotInstalled) => { - let installed = <[u8; 8]>::try_from(record.payload.as_slice()) + let Some(installed) = <[u8; 8]>::try_from(record.payload.as_slice()) .ok() - .map(u64::from_le_bytes); - if group_id.is_some() && installed == group_id { - applied_tail.clear(); - max_applied_epoch = NOT_YET_APPLIED_EPOCH; + .map(u64::from_le_bytes) + else { + continue; + }; + for (vshard_id, scan) in &mut scans { + if group_of(*vshard_id) == Some(installed) { + *scan = AppliedRecovery::empty(); + } } } Some(RecordType::CalvinApplied) => { match CalvinAppliedPayload::from_bytes(&record.payload) { - Ok(p) if p.vshard_id == vshard_id => { - accumulate( - &mut applied_tail, - &mut max_applied_epoch, - p.epoch, - p.position, - ); - } - Ok(_) => { - // Different vshard — skip. - } + Ok(p) => note(&mut scans, p.vshard_id, p.epoch, p.position), Err(e) => { warn!( lsn = record.header.lsn, @@ -112,15 +155,8 @@ fn scan_applied( } Some(RecordType::TransactionRedo) => match RedoRecord::from_bytes(&record.payload) { Ok(redo) => { - if let Some(stamp) = redo.calvin_stamp - && stamp.vshard_id == vshard_id - { - accumulate( - &mut applied_tail, - &mut max_applied_epoch, - stamp.epoch, - stamp.position, - ); + if let Some(stamp) = redo.calvin_stamp { + note(&mut scans, stamp.vshard_id, stamp.epoch, stamp.position); } } Err(e) => { @@ -134,19 +170,13 @@ fn scan_applied( _ => continue, } } - - Ok(AppliedRecovery { - // Conservative: the applied gate advances the watermark once the - // re-fan-out supplies the per-epoch expected counts. - fully_applied_epoch: NOT_YET_APPLIED_EPOCH, - applied_tail, - max_applied_epoch, - }) + Ok(scans) } -/// This vShard's applied state after a restart: the state the last -/// checkpoint saved in `catalog`, together with the markers the WAL still -/// holds. +/// Every vShard's applied state after a restart, in one pass over the WAL: +/// the state the last checkpoint saved in `catalog`, together with the +/// markers the WAL still holds. A vShard absent from the result has no +/// Calvin history on this node. /// /// A checkpoint deletes WAL segments that hold applied markers, while the /// sequencer log keeps delivering their entries after a restart. Without the @@ -154,19 +184,24 @@ fn scan_applied( /// one: its local stage refuses the rows it already wrote, the scheduler /// halts, and the transaction's completion ack never settles. /// -/// `group_id` is the data group of the vShard, when known: the markers +/// `group_of` names the data group of a vShard, when known: the markers /// before the group's last snapshot install no longer hold. -pub fn recover_applied( +pub fn recover_all_applied( wal: &WalManager, catalog: &crate::control::security::catalog::SystemCatalog, - vshard_id: u32, - group_id: Option, -) -> crate::Result { - let from_wal = scan_applied(wal, vshard_id, group_id)?; - let Some(saved) = catalog.load_calvin_applied(vshard_id)? else { - return Ok(from_wal); - }; - Ok(merge_saved(from_wal, saved.fully_applied_epoch, saved.tail)) + group_of: &dyn Fn(u32) -> Option, +) -> crate::Result> { + let mut recovered = scan_all(wal, group_of, None)?; + for saved in catalog.load_all_calvin_applied()? { + let from_wal = recovered + .remove(&saved.vshard_id) + .unwrap_or_else(AppliedRecovery::empty); + recovered.insert( + saved.vshard_id, + merge_saved(from_wal, saved.fully_applied_epoch, saved.tail), + ); + } + Ok(recovered) } /// Fold a saved `(fully_applied_epoch, tail)` into a WAL scan's result. @@ -207,21 +242,6 @@ fn record_type_of(record: &WalRecord) -> Option { RecordType::from_raw(raw_type) } -/// Feed one applied `(epoch, position)` pair into the shared tail / watermark -/// accumulation, used by both the `CalvinApplied` and `TransactionRedo` -/// (`calvin_stamp`) marker sources so the two unify into one applied set. -fn accumulate( - applied_tail: &mut BTreeSet<(u64, u32)>, - max_applied_epoch: &mut u64, - epoch: u64, - position: u32, -) { - applied_tail.insert((epoch, position)); - if *max_applied_epoch == NOT_YET_APPLIED_EPOCH || epoch > *max_applied_epoch { - *max_applied_epoch = epoch; - } -} - #[cfg(test)] mod tests { use super::*; @@ -411,15 +431,96 @@ mod tests { ]) .unwrap(); - let rec = recover_applied(&wal, &catalog, 1, None).unwrap(); + let recovered = recover_all_applied(&wal, &catalog, &|_| None).unwrap(); + let rec = &recovered[&1]; assert_eq!(rec.fully_applied_epoch, 2); assert!(rec.applied_tail.contains(&(4, 1))); assert!(rec.applied_tail.contains(&(6, 0))); assert_eq!(rec.max_applied_epoch, 6); - // A vShard with nothing saved keeps the plain WAL scan. - let other = recover_applied(&wal, &catalog, 9, None).unwrap(); - assert_eq!(other, read_applied_recovery(&wal, 9).unwrap()); + // A vShard with nothing saved and no marker has no history. + assert!(!recovered.contains_key(&9)); + } + + fn empty_catalog(dir: &TempDir) -> crate::control::security::catalog::SystemCatalog { + crate::control::security::catalog::SystemCatalog::open(&dir.path().join("system.redb")) + .unwrap() + } + + /// The one boot pass yields, for every vShard, what a scan of that vShard + /// alone yields. + #[test] + fn one_pass_recovery_matches_per_vshard_scans() { + use crate::types::VShardId; + let dir = TempDir::new().unwrap(); + let wal = open_wal(&dir); + let appender = || wal.appender(crate::wal::manager::NO_APPLY_KEY); + appender() + .append_calvin_applied(VShardId::new(1), 1, 0) + .unwrap(); + appender() + .append_calvin_applied(VShardId::new(2), 1, 1) + .unwrap(); + appender().append_snapshot_installed(10).unwrap(); + appender() + .append_calvin_applied(VShardId::new(3), 4, 0) + .unwrap(); + appender() + .append_calvin_applied(VShardId::new(1), 2, 3) + .unwrap(); + appender().append_snapshot_installed(20).unwrap(); + appender() + .append_calvin_applied(VShardId::new(2), 6, 0) + .unwrap(); + wal.sync().unwrap(); + let group_of = |vshard: u32| match vshard { + 1 => Some(10), + 2 => Some(20), + _ => None, + }; + let catalog_dir = TempDir::new().unwrap(); + let catalog = empty_catalog(&catalog_dir); + + let all = recover_all_applied(&wal, &catalog, &group_of).unwrap(); + + assert_eq!(all.keys().copied().collect::>(), vec![1, 2, 3]); + for (vshard, recovered) in &all { + let alone = scan_applied(&wal, *vshard, group_of(*vshard)).unwrap(); + assert_eq!(recovered, &alone, "vShard {vshard}"); + } + assert_eq!(all[&1].applied_tail, [(2, 3)].into_iter().collect()); + assert_eq!(all[&2].applied_tail, [(6, 0)].into_iter().collect()); + assert_eq!(all[&3].applied_tail, [(4, 0)].into_iter().collect()); + } + + /// A snapshot install record drops the markers of its own group's + /// vShards only. + #[test] + fn a_snapshot_install_record_clears_only_its_group() { + use crate::types::VShardId; + let dir = TempDir::new().unwrap(); + let wal = open_wal(&dir); + let appender = || wal.appender(crate::wal::manager::NO_APPLY_KEY); + appender() + .append_calvin_applied(VShardId::new(1), 3, 0) + .unwrap(); + appender() + .append_calvin_applied(VShardId::new(2), 3, 1) + .unwrap(); + appender().append_snapshot_installed(8).unwrap(); + appender() + .append_calvin_applied(VShardId::new(1), 5, 2) + .unwrap(); + wal.sync().unwrap(); + let group_of = |vshard: u32| if vshard == 1 { Some(8) } else { Some(9) }; + let catalog_dir = TempDir::new().unwrap(); + let catalog = empty_catalog(&catalog_dir); + + let all = recover_all_applied(&wal, &catalog, &group_of).unwrap(); + + assert_eq!(all[&1].applied_tail, [(5, 2)].into_iter().collect()); + assert_eq!(all[&1].max_applied_epoch, 5); + assert_eq!(all[&2].applied_tail, [(3, 1)].into_iter().collect()); } /// A snapshot install of the vShard's group drops every marker before diff --git a/nodedb/src/control/cluster/calvin_snapshot/capture.rs b/nodedb/src/control/cluster/calvin_snapshot/capture.rs index ff95938c9..24b66f15b 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/capture.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/capture.rs @@ -109,13 +109,13 @@ pub fn capture_calvin_cut( group_vshards: &HashSet, through: u64, ) -> GroupCalvinCut { - let mirrors = shared.authorization_fence.calvin_mirrors(); + let ledgers = &shared.calvin.applied; let mut ids: Vec = group_vshards.iter().copied().collect(); ids.sort_unstable(); let vshards = ids .into_iter() .filter_map(|vshard_id| { - let (fully_applied_epoch, tail) = mirrors.get(vshard_id)?.snapshot(); + let (fully_applied_epoch, tail) = ledgers.get(vshard_id)?.snapshot(); Some(VShardCalvinState { vshard_id, fully_applied_epoch, diff --git a/nodedb/src/control/cluster/calvin_snapshot/install.rs b/nodedb/src/control/cluster/calvin_snapshot/install.rs index b528d7cbd..878420faf 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/install.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/install.rs @@ -5,11 +5,11 @@ //! The snapshot replaced the storage of the group's vShards with the //! leader's, which holds exactly the Calvin positions the cut names. So //! each vShard's applied state here becomes the cut's: in the applied -//! mirror, which a checkpoint saves from, and in the catalog, which a -//! scheduler starts from. Its base becomes the cut's sequencer index, which -//! moves its generation: a scheduler started before the install installs -//! nothing more, and the next scheduler reconcile starts it again from the -//! installed state. +//! ledger, which a scheduler starts from and a checkpoint saves from, and +//! in the catalog, which boot recovery reads. Its base becomes the cut's +//! sequencer index, which moves its generation: a scheduler started before +//! the install installs nothing more, and the next scheduler reconcile +//! starts it again from the installed state. //! //! A WAL `SnapshotInstalled` record of the group, written before this runs, //! makes boot recovery drop the vShards' applied markers from before it. @@ -52,9 +52,12 @@ pub fn install_calvin_cut( }) .collect(); - let mirrors = shared.authorization_fence.calvin_mirrors(); for state in &states { - mirrors.register(state.vshard_id, state.fully_applied_epoch, &state.tail); + shared.calvin.applied.install( + state.vshard_id, + state.fully_applied_epoch, + state.tail.clone(), + ); } let catalog = shared.credentials.catalog(); let settle_error = |source| SnapshotInstallError::Settle { diff --git a/nodedb/src/control/cluster/calvin_snapshot/requirement.rs b/nodedb/src/control/cluster/calvin_snapshot/requirement.rs index 8a47eb1b5..a337335f8 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/requirement.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/requirement.rs @@ -18,6 +18,10 @@ //! sequencer log, and then keeps the base (see [`may_start`]). //! - The sequencer log keeps the range a waiting vShard will replay (see //! [`crate::control::state::CalvinBases::replay_floor`]). +//! +//! A data group whose chunked redo streams this node dropped when it left +//! the group requires a snapshot too, until one installs (see +//! `wal_replication::transaction_redo::chunks::owed`). use std::collections::BTreeSet; use std::sync::{Arc, Mutex, RwLock}; @@ -27,8 +31,9 @@ use nodedb_cluster::multi_raft::MultiRaft; use crate::control::security::catalog::calvin_base::CalvinBase; use crate::control::state::SharedState; -/// Load this node's Calvin bases and install the data-group snapshot -/// requirement on `multi_raft`, before its data groups take entries. +/// Load this node's Calvin bases and owed redo snapshots, and install the +/// data-group snapshot requirement on `multi_raft`, before its data groups +/// take entries. pub fn install_snapshot_requirement( shared: &Arc, routing: Arc>, @@ -39,6 +44,9 @@ pub fn install_snapshot_requirement( catalog.load_calvin_bases()?, catalog.load_calvin_sequencer_install()?, ); + shared + .redo_chunks + .load_owed(catalog.load_redo_snapshot_owed()?); let weak = Arc::downgrade(shared); multi_raft.set_snapshot_requirement(Arc::new(move |group_id, sequencer_first| { let Some(shared) = weak.upgrade() else { @@ -48,7 +56,9 @@ pub fn install_snapshot_requirement( .read() .unwrap_or_else(|p| p.into_inner()) .vshards_for_group(group_id); - !shared.calvin.bases.group_reaches(&vshards, sequencer_first) + // Evaluated first and always: it registers the vShards as waiting. + let reaches = shared.calvin.bases.group_reaches(&vshards, sequencer_first); + !reaches || shared.redo_chunks.owes_snapshot(group_id) })); Ok(()) } @@ -161,7 +171,6 @@ pub fn forget_left(shared: &SharedState, vshards: &[u32]) { return; } let catalog = shared.credentials.catalog(); - let mirrors = shared.authorization_fence.calvin_mirrors(); let resets: Vec = vshards .iter() @@ -175,7 +184,7 @@ pub fn forget_left(shared: &SharedState, vshards: &[u32]) { }) .collect(); for &vshard_id in vshards { - mirrors.remove(vshard_id); + shared.calvin.applied.remove(vshard_id); } let forgotten = catalog .replace_calvin_applied(&resets) diff --git a/nodedb/src/control/cluster/metadata_applier/boot_seed.rs b/nodedb/src/control/cluster/metadata_applier/boot_seed.rs index 9df6fa735..98ca9f3dc 100644 --- a/nodedb/src/control/cluster/metadata_applier/boot_seed.rs +++ b/nodedb/src/control/cluster/metadata_applier/boot_seed.rs @@ -7,7 +7,7 @@ //! loads those rows into the in-memory state before any entry applies. //! State that no row holds starts fresh: //! - `last_applied_hlc` is the highest expiry over the seeded leases and drains. -//! - `metadata_ddl_applied_token` is 0. +//! - `metadata_ddl.applied_token` is 0. //! - `topology_log`, `routing_log`, and `catalog_entries_applied` are empty. //! //! A read error fails boot: running with a partial view of leases or drains @@ -45,7 +45,8 @@ pub fn seed_host_tables(shared: &SharedState) -> crate::Result<()> { .insert(record.token, record.objects, record.proposed_at); } *shared - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .unwrap_or_else(|p| p.into_inner()) = owner.map( @@ -240,7 +241,8 @@ mod tests { assert_eq!(record.proposed_at, Hlc::new(60, 0)); assert_eq!( state - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .unwrap() .map(|owner| (owner.token, owner.node_id)), @@ -248,7 +250,8 @@ mod tests { ); assert_eq!( state - .metadata_ddl_applied_token + .metadata_ddl + .applied_token .load(std::sync::atomic::Ordering::Acquire), 0 ); diff --git a/nodedb/src/control/cluster/metadata_applier/dispatch.rs b/nodedb/src/control/cluster/metadata_applier/dispatch.rs index 34dabf8c1..aece97cff 100644 --- a/nodedb/src/control/cluster/metadata_applier/dispatch.rs +++ b/nodedb/src/control/cluster/metadata_applier/dispatch.rs @@ -261,7 +261,8 @@ impl MetadataCommitApplier { } self.apply_host_side_effects(entry, raft_index).await?; shared - .metadata_ddl_applied_token + .metadata_ddl + .applied_token .store(token, std::sync::atomic::Ordering::Release); Ok(()) } @@ -509,7 +510,7 @@ mod tests { /// `DdlPrepareAcquire` does. A pending propose and finalize apply only /// under the owner's token. fn own_ddl_lease(state: &SharedState, token: u64) { - *state.metadata_ddl_owner.lock().unwrap() = + *state.metadata_ddl.owner.lock().unwrap() = Some(crate::control::metadata_proposer::DdlPrepareOwner { token, node_id: state.node_id, @@ -564,7 +565,8 @@ mod tests { } assert_ne!( state - .metadata_ddl_applied_token + .metadata_ddl + .applied_token .load(std::sync::atomic::Ordering::Acquire), dead, "the dead owner's proposer must see its entries superseded" diff --git a/nodedb/src/control/cluster/metadata_applier/host_state.rs b/nodedb/src/control/cluster/metadata_applier/host_state.rs index c47f7d6f6..8177d65f8 100644 --- a/nodedb/src/control/cluster/metadata_applier/host_state.rs +++ b/nodedb/src/control/cluster/metadata_applier/host_state.rs @@ -32,7 +32,8 @@ impl MetadataCommitApplier { ) -> Result<(), crate::Error> { let shared = self.shared_state()?; let mut owner = shared - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); if owner.is_none() { @@ -53,7 +54,8 @@ impl MetadataCommitApplier { pub(super) fn apply_ddl_prepare_release(&self, token: u64) -> Result<(), crate::Error> { let shared = self.shared_state()?; let mut owner = shared - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); if owner.is_some_and(|current| current.token == token) { diff --git a/nodedb/src/control/cluster/metadata_applier/pending_ddl.rs b/nodedb/src/control/cluster/metadata_applier/pending_ddl.rs index c9a415c0d..acc8563f9 100644 --- a/nodedb/src/control/cluster/metadata_applier/pending_ddl.rs +++ b/nodedb/src/control/cluster/metadata_applier/pending_ddl.rs @@ -78,7 +78,7 @@ impl MetadataCommitApplier { /// for the next re-delivery instead of silently skipping the rest. /// /// Applies only while `token` owns the preparation lease. A finalize - /// that applies records `token` in `metadata_ddl_applied_token`, which + /// that applies records `token` in `metadata_ddl.applied_token`, which /// is how its proposer learns the objects landed. pub(super) async fn apply_ddl_pending_finalize( &self, @@ -104,7 +104,8 @@ impl MetadataCommitApplier { self.credentials.catalog().remove_pending_ddl(token)?; shared.pending_ddl.take(token); shared - .metadata_ddl_applied_token + .metadata_ddl + .applied_token .store(token, std::sync::atomic::Ordering::Release); Ok(()) } diff --git a/nodedb/src/control/cluster/mod.rs b/nodedb/src/control/cluster/mod.rs index 8f8e0abdf..de6a5717e 100644 --- a/nodedb/src/control/cluster/mod.rs +++ b/nodedb/src/control/cluster/mod.rs @@ -27,6 +27,7 @@ pub mod metadata_stamp; pub mod pem_io; pub mod read_index; pub mod recovery_check; +pub mod redo_stream_membership; pub mod sequencer_compaction; pub mod sequencer_halt; pub mod sequencer_snapshot; diff --git a/nodedb/src/control/cluster/redo_stream_membership.rs b/nodedb/src/control/cluster/redo_stream_membership.rs new file mode 100644 index 000000000..fdcc58ee0 --- /dev/null +++ b/nodedb/src/control/cluster/redo_stream_membership.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The membership pass over this node's chunked redo streams. +//! +//! - A data group this node no longer mounts gets no later entry here, so +//! no final entry, abandon, or later term closes its streams. They drop +//! here, with their WAL floor holds. The group then owes a snapshot +//! install before it applies here again. A remount resumes its old log, +//! which can still name a dropped stream. +//! - A mounted group that owes a snapshot refuses log entries until one +//! installs, and its leadership here moves to a replica that can lead. +//! This covers a group that mounted again, and a group whose apply here +//! held an entry of a dropped stream. + +use std::sync::Mutex; + +use nodedb_cluster::multi_raft::MultiRaft; + +use crate::control::state::SharedState; + +/// Run the pass once. The membership reconcile calls it on every tick. +pub fn reconcile_redo_streams(shared: &SharedState, multi_raft: &Mutex) { + drop_unmounted(shared, multi_raft); + require_owed_snapshots(shared, multi_raft); +} + +/// Drop the streams of every data group this node no longer mounts. +/// +/// The `MultiRaft` lock is held throughout. A group that mounts meanwhile +/// keeps its streams, and a remount sees the group's debt. The debt write +/// is a catalog commit under that lock. It runs only when a left group +/// still holds a stream. +fn drop_unmounted(shared: &SharedState, multi_raft: &Mutex) { + let dropped = { + let multi_raft = multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + shared.redo_chunks.drop_unhosted_groups( + |group_id| multi_raft.contains_group(group_id), + shared.credentials.catalog(), + ) + }; + match dropped { + Ok(0) => {} + Ok(dropped) => tracing::info!( + dropped, + "redo chunks: dropped the open streams of data groups this node left; each such \ + group installs a snapshot before it applies here again" + ), + Err(error) => { + tracing::error!( + %error, + "redo chunks: the snapshot debt of data groups this node left did not \ + persist; their streams stay until a later pass records it" + ); + crate::diag::redo_snapshot_debt_not_recorded(&error); + } + } +} + +/// Make every mounted group that owes a snapshot refuse log entries, and +/// move a leadership of it this node holds: a leader takes no snapshot. +/// +/// The debt is read under the `MultiRaft` lock. An install settles the debt +/// before it refreshes the group's requirement under the same lock, so this +/// pass never raises a requirement an install has paid. +fn require_owed_snapshots(shared: &SharedState, multi_raft: &Mutex) { + let owed = shared.redo_chunks.owed_groups(); + if owed.is_empty() { + return; + } + let mut multi_raft = multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + for group_id in owed { + if !multi_raft.contains_group(group_id) || !shared.redo_chunks.owes_snapshot(group_id) { + continue; + } + if !multi_raft.snapshot_required(group_id) { + multi_raft.set_snapshot_required(group_id, true); + tracing::warn!( + group_id, + "redo chunks: the group's replica here owes a snapshot install; it refuses \ + log entries until one installs" + ); + } + if let Some(target) = multi_raft.hand_off_leadership(group_id) { + tracing::info!( + group_id, + target, + "redo chunks: the group's replica here owes a snapshot install; its \ + leadership moves to a replica that can lead" + ); + } + } +} diff --git a/nodedb/src/control/cluster/snapshot_applier.rs b/nodedb/src/control/cluster/snapshot_applier.rs index e75512b64..035c04813 100644 --- a/nodedb/src/control/cluster/snapshot_applier.rs +++ b/nodedb/src/control/cluster/snapshot_applier.rs @@ -167,6 +167,7 @@ impl DataPlaneSnapshotApplier { cut_index, event_lane, calvin, + redo_streams, } = split_by_core(group_id, snap, &cores)?; // Refused before any core changes: storage installed without its // Calvin cut will hold Calvin transactions no applied state names. @@ -214,6 +215,17 @@ impl DataPlaneSnapshotApplier { &group_vshards, calvin, )?; + // After the install marker too: boot drops the group's earlier streams + // there and rebuilds these. + self.shared + .redo_chunks + .install_group(group_id, redo_streams) + .await + .map_err(|source| SnapshotInstallError::Settle { + group_id, + step: SettleStep::RedoStreams, + source: source.into(), + })?; crate::control::pitr::force_base_after_install(&self.shared); self.settle( group_id, @@ -243,6 +255,17 @@ impl DataPlaneSnapshotApplier { if let Some(tracker) = self.shared.propose_tracker.get() { tracker.cover_through(group_id, cut_index); } + // Last: the group's streams here are the snapshot's now, so a + // snapshot this node owed the group is paid. An install that fails + // earlier leaves the debt, and the group keeps requiring one. + self.shared + .redo_chunks + .settle_owed(self.shared.credentials.catalog(), group_id) + .map_err(|source| SnapshotInstallError::Settle { + group_id, + step: SettleStep::RedoStreams, + source, + })?; Ok(()) } diff --git a/nodedb/src/control/cluster/snapshot_builder.rs b/nodedb/src/control/cluster/snapshot_builder.rs index debd6ddcd..050590b8f 100644 --- a/nodedb/src/control/cluster/snapshot_builder.rs +++ b/nodedb/src/control/cluster/snapshot_builder.rs @@ -424,6 +424,9 @@ impl nodedb_cluster::SnapshotBuilder for DataPlaneSnapshotBuilder { } merged.group_cut_index = cut_index; merged.group_calvin = Some(calvin_cut); + // Read under the fence, so the streams hold the chunks of the entries + // at or below the cut. + merged.group_redo_streams = self.shared.redo_chunks.capture_group(group_id); merged.group_event_lane = lane_snapshot::capture(&self.shared, &group_vshards) .await .map_err(|e| Box::new(e) as Box)?; diff --git a/nodedb/src/control/cluster/snapshot_install/error.rs b/nodedb/src/control/cluster/snapshot_install/error.rs index f07b043c5..a433c4b04 100644 --- a/nodedb/src/control/cluster/snapshot_install/error.rs +++ b/nodedb/src/control/cluster/snapshot_install/error.rs @@ -16,6 +16,8 @@ pub enum SettleStep { EventLane, /// Replacing the group vShards' Calvin applied state and base. CalvinState, + /// Replacing the group's open chunked redo streams. + RedoStreams, } impl std::fmt::Display for SettleStep { @@ -26,6 +28,7 @@ impl std::fmt::Display for SettleStep { Self::ProposalKeys => "proposal-key persist", Self::EventLane => "event lane install", Self::CalvinState => "Calvin state install", + Self::RedoStreams => "redo stream install", }) } } diff --git a/nodedb/src/control/cluster/snapshot_install/split.rs b/nodedb/src/control/cluster/snapshot_install/split.rs index b059a7f97..8a0488705 100644 --- a/nodedb/src/control/cluster/snapshot_install/split.rs +++ b/nodedb/src/control/cluster/snapshot_install/split.rs @@ -90,6 +90,8 @@ pub struct CoreShares { pub event_lane: crate::types::snapshot::GroupEventLane, /// The Calvin cut of the group's vShards. pub calvin: Option, + /// The group's open chunked redo streams. + pub redo_streams: Vec, } /// `(database, collection)` of a `"{db}:{tid}:{collection}[:suffix]"` key. @@ -211,6 +213,7 @@ pub fn split_by_core( group_cut_index, group_event_lane, group_calvin, + group_redo_streams, } = snap; let mut s = Splitter { @@ -346,6 +349,7 @@ pub fn split_by_core( cut_index: group_cut_index, event_lane: group_event_lane, calvin: group_calvin, + redo_streams: group_redo_streams, }) } diff --git a/nodedb/src/control/cluster/start_raft/core.rs b/nodedb/src/control/cluster/start_raft/core.rs index 0e4d5d2e7..08bd79e3e 100644 --- a/nodedb/src/control/cluster/start_raft/core.rs +++ b/nodedb/src/control/cluster/start_raft/core.rs @@ -69,6 +69,9 @@ pub async fn start_raft( loop_build.calvin_read_result_senders, loop_build.sequencer_state_machine, )?; + // The async proposer is installed: a failed redo stream abandon can be + // retried from here on. + crate::control::wal_replication::transaction_redo::spawn_redo_abandoner(&shared); let ready_rx = finish_observability( handle, diff --git a/nodedb/src/control/cluster/start_raft/hooks.rs b/nodedb/src/control/cluster/start_raft/hooks.rs index bf5875f2e..e4bde7639 100644 --- a/nodedb/src/control/cluster/start_raft/hooks.rs +++ b/nodedb/src/control/cluster/start_raft/hooks.rs @@ -80,6 +80,29 @@ pub(super) async fn build_hooks( multi_raft, )?; + // Every vShard's Calvin applied ledger, from one WAL pass and the state + // the last checkpoint saved. The data-group apply loop and the + // schedulers read the ledgers, so they fill before either starts. A + // boot completion of a snapshot install below replaces its group's. + let vshard_to_group = handle + .routing + .read() + .unwrap_or_else(|p| p.into_inner()) + .vshard_to_group() + .to_vec(); + let group_of = |vshard_id: u32| vshard_to_group.get(vshard_id as usize).copied(); + let recovered = crate::control::cluster::calvin::scheduler::recover_all_applied( + &shared.wal, + shared.credentials.catalog(), + &group_of, + )?; + for (vshard_id, state) in recovered { + shared + .calvin + .applied + .install(vshard_id, state.fully_applied_epoch, state.applied_tail); + } + // Per-group snapshot builder for the SEND path: on the leader, build the // real serialized engine state for a lagging follower's group vshards // (replacing the prior empty stub bytes). diff --git a/nodedb/src/control/cluster/start_raft/proposer_wiring.rs b/nodedb/src/control/cluster/start_raft/proposer_wiring.rs index 26b71c031..b45846901 100644 --- a/nodedb/src/control/cluster/start_raft/proposer_wiring.rs +++ b/nodedb/src/control/cluster/start_raft/proposer_wiring.rs @@ -116,13 +116,13 @@ fn install_compactor( .ok_or_else(|| crate::Error::Internal { detail: "raft log compaction: shared state dropped".into(), })?; - let mirrors = shared.authorization_fence.calvin_mirrors(); + let ledgers = &shared.calvin.applied; let mut sm = sm_for_compact.lock().unwrap_or_else(|p| p.into_inner()); let undurable = sm.undurable_floor(|vshard, epoch, position| { - // A vShard with no scheduler here holds no state to lose. - mirrors + // A vShard with no ledger here holds no state to lose. + ledgers .get(vshard) - .is_none_or(|mirror| mirror.is_applied(epoch, position)) + .is_none_or(|ledger| ledger.is_applied(epoch, position)) }); let floor = [ sm.min_catch_up_from(), diff --git a/nodedb/src/control/cluster/start_raft_helpers.rs b/nodedb/src/control/cluster/start_raft_helpers.rs index 067464de0..1642ad138 100644 --- a/nodedb/src/control/cluster/start_raft_helpers.rs +++ b/nodedb/src/control/cluster/start_raft_helpers.rs @@ -5,7 +5,6 @@ use std::sync::{Arc, Mutex, RwLock}; use nodedb_cluster::calvin::{CalvinCompletionRegistry, SequencerStateMachine}; use crate::control::cluster::calvin::scheduler::metrics::SchedulerMetrics; -use crate::control::cluster::calvin::scheduler::recover_applied; use crate::control::cluster::calvin::{ RaftSequencerProposer, ReadResultEvent, Scheduler, SchedulerConfig, SchedulerParams, SequencerProposer, @@ -159,6 +158,10 @@ fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate:: calvin_completion_registry, ); calvin_snapshot::retain_mounted(shared, raft_loop_handle, routing); + crate::control::cluster::redo_stream_membership::reconcile_redo_streams( + shared, + raft_loop_handle, + ); let mut spawned = 0usize; let mut kept = Vec::new(); for vshard_id in hosted { @@ -198,28 +201,11 @@ fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate:: continue; }; - // The applied state the last checkpoint saved, with the markers the - // WAL still holds: a checkpoint deletes the segments that held older - // markers, and the sequencer log delivers their entries again. - // A failed read leaves the vShard for the next pass. The pass goes - // on, so the schedulers it started keep their bases. - let recovery = match recover_applied( - &shared.wal, - shared.credentials.catalog(), - vshard_id, - group_id, - ) { - Ok(recovery) => recovery, - Err(error) => { - tracing::warn!( - vshard_id, - %error, - "calvin: the vShard's applied state did not load; its scheduler starts \ - on a later pass" - ); - continue; - } - }; + // The vShard's applied ledger: boot recovery or a snapshot install + // filled it, and a vShard with no Calvin history here gets an empty + // one. The scheduler rebuilds up to its highest applied epoch. + let ledger = shared.calvin.applied.get_or_create(vshard_id); + let rebuild_target_epoch = ledger.max_applied_epoch(); let (sequenced_tx, sequenced_rx) = tokio::sync::mpsc::channel(scheduler_config.channel_capacity); @@ -299,9 +285,8 @@ fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate:: multi_raft: raft_loop_handle.clone(), sequencer_proposer: Arc::clone(sequencer_proposer), sequencer_state_machine: Arc::clone(sequencer_state_machine), - fully_applied_epoch: recovery.fully_applied_epoch, - applied_tail: recovery.applied_tail, - rebuild_target_epoch: recovery.max_applied_epoch, + ledger, + rebuild_target_epoch, config: scheduler_config.clone(), metrics: SchedulerMetrics::new(), read_result_rx, @@ -368,7 +353,9 @@ pub(super) struct SpawnVshardSchedulersParams<'a> { /// (covers the bootstrap node, which already sees its membership) and then /// spawns a background task that re-reconciles on a short interval until /// shutdown. Reconcile is idempotent: it starts schedulers for vShards this -/// node gained and stops those of vShards it left. +/// node gained and stops those of vShards it left. Each pass also runs the +/// membership pass over this node's chunked redo streams (see +/// [`crate::control::cluster::redo_stream_membership`]). pub(super) fn spawn_vshard_schedulers( params: SpawnVshardSchedulersParams<'_>, ) -> crate::Result<()> { diff --git a/nodedb/src/control/distributed_applier/applied_index.rs b/nodedb/src/control/distributed_applier/applied_index.rs index 6ab3c790c..7a9314dc2 100644 --- a/nodedb/src/control/distributed_applier/applied_index.rs +++ b/nodedb/src/control/distributed_applier/applied_index.rs @@ -83,7 +83,13 @@ pub fn save_applied_index(state: &Arc, group_id: u64, applied_index #[derive(Debug, Default)] pub struct AppliedPrefix { floor: Option, - broken: bool, + /// `(first, last)` non-durable entries above `covered_through`, when any. + /// `first` can sit below the true first break, never above it. + breaks: Option<(u64, u64)>, + /// The highest index recorded above `covered_through`. + last_recorded: Option, + /// An installed snapshot holds every entry at or below this index. + covered_through: u64, } impl AppliedPrefix { @@ -96,19 +102,55 @@ impl AppliedPrefix { /// /// A failure breaks the prefix permanently for this batch: every later /// entry, successful or not, is past the gap and can no longer be covered - /// by the floor. Call this ONLY for entries whose success means "this - /// entry's redo record is WAL-fsync-durable" — branches that apply no + /// by the floor. Only [`Self::cover_through`] heals a break. Call this + /// ONLY for entries whose success means "this entry's redo record is + /// WAL-fsync-durable" — branches that apply no /// durable state must not call it at all (see [`Self::skip`]). pub fn record(&mut self, index: u64, applied_ok: bool) { + // The installed snapshot holds the entry's state, whatever its apply did. + if index <= self.covered_through { + return; + } + self.last_recorded = Some(index); if !applied_ok { - self.broken = true; + self.breaks = Some(match self.breaks { + None => (index, index), + Some((first, _)) => (first, index), + }); return; } - if !self.broken { + if self.breaks.is_none() { self.floor = Some(index); } } + /// Count every entry through `index` durable: an installed snapshot holds + /// their state. + /// + /// A break at or below `index` heals. A break above it stays. A later + /// record at or below `index` changes nothing. + pub fn cover_through(&mut self, index: u64) { + if index <= self.covered_through { + return; + } + self.covered_through = index; + self.breaks = match self.breaks { + Some((_, last)) if last <= index => None, + // The breaks above `index` start no lower than `index + 1`. + Some((first, last)) if first <= index => Some((index.saturating_add(1), last)), + unchanged => unchanged, + }; + // Every record above `index` came after the last healed break, so + // each one was durable when no break remains. + let reach = match self.breaks { + None => self.last_recorded.map_or(index, |last| last.max(index)), + Some(_) => index, + }; + if self.floor.is_none_or(|floor| floor < reach) { + self.floor = Some(reach); + } + } + /// Note an entry that carries no durable state — it neither advances the /// prefix nor breaks it. /// @@ -188,4 +230,51 @@ mod tests { broken.record(4, true); assert_eq!(broken.floor(), Some(1)); } + + #[test] + fn a_covering_snapshot_heals_a_break_at_or_below_it() { + let mut prefix = AppliedPrefix::new(); + prefix.record(1, true); + prefix.record(2, false); + prefix.record(3, true); + assert_eq!(prefix.floor(), Some(1)); + + prefix.cover_through(5); + assert_eq!(prefix.floor(), Some(5)); + prefix.record(4, false); + assert_eq!(prefix.floor(), Some(5), "a covered entry cannot break"); + prefix.record(6, true); + assert_eq!(prefix.floor(), Some(6)); + } + + #[test] + fn a_break_above_the_snapshot_still_holds_the_floor() { + let mut prefix = AppliedPrefix::new(); + prefix.record(1, true); + prefix.record(7, false); + prefix.cover_through(5); + assert_eq!(prefix.floor(), Some(5)); + prefix.record(8, true); + assert_eq!(prefix.floor(), Some(5), "entry 7 is not durable"); + + // Breaks on both sides: the one above the snapshot still holds. + let mut split = AppliedPrefix::new(); + split.record(2, false); + split.record(9, false); + split.cover_through(5); + split.record(10, true); + assert_eq!(split.floor(), Some(5)); + split.cover_through(9); + assert_eq!(split.floor(), Some(10), "entry 10 settled durable"); + } + + #[test] + fn a_lower_cover_changes_nothing() { + let mut prefix = AppliedPrefix::new(); + prefix.cover_through(5); + prefix.record(6, false); + prefix.cover_through(4); + prefix.record(7, true); + assert_eq!(prefix.floor(), Some(5)); + } } diff --git a/nodedb/src/control/distributed_applier/apply_loop/lane.rs b/nodedb/src/control/distributed_applier/apply_loop/lane.rs index 71728e455..e45dbe55e 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/lane.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/lane.rs @@ -63,6 +63,8 @@ impl QueuedEntry { ReplicatedWrite::CutBarrier { .. } | ReplicatedWrite::CalvinReadResult { .. } | ReplicatedWrite::SurrogateBind { .. } + | ReplicatedWrite::RedoChunk { .. } + | ReplicatedWrite::RedoAbandon { .. } ) { return None; @@ -85,7 +87,9 @@ impl QueuedEntry { ReplicatedWrite::ArraySchema { .. } | ReplicatedWrite::CutBarrier { .. } | ReplicatedWrite::CalvinReadResult { .. } - | ReplicatedWrite::SurrogateBind { .. } => false, + | ReplicatedWrite::SurrogateBind { .. } + | ReplicatedWrite::RedoChunk { .. } + | ReplicatedWrite::RedoAbandon { .. } => false, _ => matches!( decode_replicated_entry(&self.entry.data), Ok(Some((_, _, plan, _))) if plan_writes_user_data(&plan) @@ -99,7 +103,8 @@ impl QueuedEntry { /// and a schema import must follow every earlier entry's apply. A topic /// publication awaits its proposal marker's fsync. A cut barrier /// persists its floor, and a capturing one snapshots its tenants, after - /// every earlier entry and before every later one. + /// every earlier entry and before every later one. A redo chunk or + /// abandon changes the stream a later entry of its group reads. pub fn is_exclusive(&self) -> bool { self.decoded.as_ref().is_some_and(|e| { matches!( @@ -110,6 +115,8 @@ impl QueuedEntry { | ReplicatedWrite::ArrayCellDelete { .. } | ReplicatedWrite::TopicPublish { .. } | ReplicatedWrite::CutBarrier { .. } + | ReplicatedWrite::RedoChunk { .. } + | ReplicatedWrite::RedoAbandon { .. } ) }) } @@ -153,8 +160,8 @@ pub(super) struct Lane { /// runs, or an exclusive entry that runs. pub blocking: Option, /// The durable prefix over every entry this process settled for the - /// group. A break holds for the life of the process: an index saved past - /// a non-durable entry lets the next boot skip it. + /// group. A break holds until a snapshot install covers it: an index + /// saved past a non-durable entry lets the next boot skip it. prefix: AppliedPrefix, /// The floor last saved for the group. saved_floor: Option, @@ -326,6 +333,17 @@ impl Lane { settled } + /// Count every entry of the group through `installed_through` durable. + /// An installed snapshot holds their state, so a break at or below it + /// heals. + /// + /// The caller passes the index it read under the group's apply gate, + /// before it admits an entry. Every entry above the index reaches the + /// lane after the install, so none settles before this call. + pub fn cover_through(&mut self, installed_through: u64) { + self.prefix.cover_through(installed_through); + } + /// Whether the durable floor moved past the floor last saved. pub fn floor_pending(&self) -> bool { self.prefix @@ -410,6 +428,48 @@ mod tests { ); } + #[test] + fn a_covering_snapshot_lets_the_floor_pass_a_held_entry() { + let tracker = ProposeTracker::new(); + let marks = TenantMarks::default(); + let mut lane = Lane::new(1); + lane.push(slot(1, SlotState::Concluded(PrefixStep::Record(true)))); + lane.push(slot(2, SlotState::Concluded(PrefixStep::Record(false)))); + lane.push(slot(3, SlotState::Running)); + lane.settle(&tracker, &marks); + assert_eq!(lane.take_floor_to_save(), Some(1)); + + // A snapshot at index 4 installs while entry 3 still runs. + lane.cover_through(4); + assert_eq!(lane.take_floor_to_save(), Some(4)); + + assert!(lane.conclude(3, false, |_| PrefixStep::Record(false))); + lane.push(slot(5, SlotState::Concluded(PrefixStep::Record(true)))); + lane.settle(&tracker, &marks); + assert_eq!( + lane.take_floor_to_save(), + Some(5), + "the snapshot holds entries 2 and 3" + ); + } + + #[test] + fn a_held_entry_above_the_snapshot_still_holds_the_floor() { + let tracker = ProposeTracker::new(); + let marks = TenantMarks::default(); + let mut lane = Lane::new(1); + lane.cover_through(4); + lane.push(slot(5, SlotState::Concluded(PrefixStep::Record(false)))); + lane.push(slot(6, SlotState::Concluded(PrefixStep::Record(true)))); + lane.settle(&tracker, &marks); + assert_eq!(lane.take_floor_to_save(), Some(4)); + assert_eq!( + lane.take_floor_to_save(), + None, + "a floor past entry 5 would let the next boot skip it" + ); + } + #[test] fn an_enqueue_that_returns_unblocks_the_next_start() { let tracker = ProposeTracker::new(); diff --git a/nodedb/src/control/distributed_applier/apply_loop/mod.rs b/nodedb/src/control/distributed_applier/apply_loop/mod.rs index 00d25c0f8..0c6e74a43 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/mod.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/mod.rs @@ -26,6 +26,8 @@ //! - [`write_dispatch`]: the generic decode + write-funnel enqueue path. //! - [`transaction_redo`]: a committed transaction's redo, stamped with its //! Raft entry and applied through the WAL replay arms. +//! - [`redo_chunk`]: the chunks, the abandon and the final entry of a +//! chunked redo stream. //! - [`topic_publish`]: a committed durable-topic publication, appended to //! this replica's topic log at the entry's position. //! - [`proposal_gate`]: skips a second committed copy of an applied proposal @@ -54,6 +56,7 @@ mod lane; mod metadata_floor; mod pipeline; mod proposal_gate; +mod redo_chunk; mod snapshot_gate; mod start; mod surrogate_bind; diff --git a/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs b/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs index e2aa68472..bfe988027 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs @@ -193,7 +193,13 @@ impl<'a> Pipeline<'a> { let log_index = queued.entry.index; let proposal_key = queued.proposal_key(); let covered_through = self.ctx.tracker.covered_through(group_id); - let permit = match admit_entry(gates, group_id, log_index, covered_through) { + let admission = admit_entry(gates, group_id, log_index, covered_through); + // Before the entry joins the lane: no entry above the installed + // index settles before the lane covers it. + if let Some(lane) = self.lanes.get_mut(&group_id) { + lane.cover_through(admission.installed_through); + } + let permit = match admission.decision { EntryAdmission::Installing => { if let Some(lane) = self.lanes.get_mut(&group_id) { lane.backlog.push_front(queued); @@ -256,6 +262,11 @@ impl<'a> Pipeline<'a> { proposal_key, metadata_floor: queued.metadata_floor(), }; + // An entry of a later term ends every earlier term of the group: + // no earlier term adds a chunk or a final entry after it. + shared + .redo_chunks + .end_terms_before(group_id, queued.entry.term); let prepared = hold_for_metadata( self.ctx.state, self.ctx.tracker, diff --git a/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs b/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs new file mode 100644 index 000000000..44cdfe355 --- /dev/null +++ b/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs @@ -0,0 +1,403 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Apply path for the entries of a chunked redo stream: a `RedoChunk`, a +//! `RedoAbandon`, and the final `TransactionRedo` entry that names the +//! stream. +//! +//! Every replica decides each of them from the log alone (see +//! `wal_replication::transaction_redo::chunks`). A chunk and an abandon +//! apply with nothing else of their group in flight, so the stream a later +//! entry reads is the one every earlier entry left. A refusal applied +//! nothing on any replica, so it advances the durable prefix like a success. +//! +//! A group whose replica here owes a snapshot install is the exception (see +//! `chunks::owed`). This node dropped streams of it that the other replicas +//! still hold. A refusal there can be this replica's alone, so it holds the +//! group's durable floor until the snapshot installs. + +use crate::control::array_sync::raft_apply::AppliedPosition; +use crate::control::distributed_applier::propose_tracker::{AppliedWrite, ProposeTracker}; +use crate::control::wal_replication::decode::ChunkedRedo; +use crate::control::wal_replication::transaction_redo::RedoTarget; +use crate::control::wal_replication::transaction_redo::chunks::{ChunkApply, RedoChunkError}; +use crate::control::wal_replication::types::RedoContent; +use crate::control::wal_replication::{CollectionIncarnation, ReplicatedEntry, ReplicatedWrite}; +use crate::types::{DatabaseId, TenantId, VShardId}; +use crate::wal::RedoStreamId; +use crate::wal::manager::RedoChunkPlacement; + +use super::context::{ApplyContext, FinishedApply}; +use super::proposal_gate::{EntryOutcome, ledger_outcome}; +use super::start::Prepared; +use super::transaction_redo::{conclude_held, enqueue_payload}; + +/// Prepare a committed `RedoChunk` or `RedoAbandon` entry of term `term`. +pub(super) fn prepare_stream_entry<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + term: u64, + entry: ReplicatedEntry, +) -> Prepared<'a> { + let placement = RedoChunkPlacement { + tenant_id: TenantId::new(entry.tenant_id), + vshard_id: VShardId::new(entry.vshard_id), + database_id: DatabaseId::new(entry.database_id), + }; + match entry.write { + ReplicatedWrite::RedoChunk { + stream, + index, + len, + bytes, + } => { + let chunk = ChunkApply { + group_id: pos.group_id, + term, + apply_key: pos.applied_key, + placement, + stream, + index, + len, + bytes, + }; + Prepared::Exclusive(Box::pin(async move { + let applied = ctx.state.redo_chunks.apply_chunk(chunk).await; + // The chunk's records are durable here, its entry not yet + // concluded. + crate::fail_point!("redo_chunk::after_chunk_durable"); + finished(ctx, pos, stream, applied) + })) + } + ReplicatedWrite::RedoAbandon { stream } => Prepared::Exclusive(Box::pin(async move { + let abandoned = ctx + .state + .redo_chunks + .abandon(stream, pos.applied_key, placement) + .await; + finished(ctx, pos, stream, abandoned) + })), + other => conclude_held( + ctx, + pos, + crate::Error::Internal { + detail: format!("redo stream apply called with another write: {other:?}"), + }, + None, + ), + } +} + +/// Resolve a chunk or abandon entry of `stream`'s waiter and conclude it. +fn finished( + ctx: ApplyContext<'_>, + pos: AppliedPosition, + stream: RedoStreamId, + applied: Result<(), RedoChunkError>, +) -> FinishedApply { + let outcome = match applied { + Err(refusal) + if refusal.is_refusal() && ctx.state.redo_chunks.owes_snapshot(pos.group_id) => + { + let error = stream_lost_here(pos, stream, LostEntry::Chunk, &refusal); + held(ctx.tracker, pos, error) + } + applied => concluded(ctx.tracker, pos, applied), + }; + FinishedApply { + group_id: pos.group_id, + log_index: pos.log_index, + outcome, + } +} + +/// Conclude a chunk or abandon entry by its stream's verdict: a success or +/// a refusal passes the entry, a storage error holds it. +fn concluded( + tracker: &ProposeTracker, + pos: AppliedPosition, + applied: Result<(), RedoChunkError>, +) -> EntryOutcome { + let durable = applied + .as_ref() + .err() + .is_none_or(RedoChunkError::is_refusal); + let result = applied + .map(|()| AppliedWrite::unversioned(Vec::new())) + .map_err(crate::Error::from); + let outcome = EntryOutcome::Applied { + durable, + result: Some(ledger_outcome(&result)), + }; + tracker.complete(pos.group_id, pos.log_index, pos.applied_key, result); + outcome +} + +/// Hold a chunk entry this replica cannot decide as the other replicas do. +/// No ledger outcome records it, so a re-delivery applies it again. +fn held(tracker: &ProposeTracker, pos: AppliedPosition, error: crate::Error) -> EntryOutcome { + tracker.complete(pos.group_id, pos.log_index, pos.applied_key, Err(error)); + EntryOutcome::Applied { + durable: false, + result: None, + } +} + +/// Which entry of a stream refused in a group that owes a snapshot. +#[derive(Debug, Clone, Copy)] +enum LostEntry { + Chunk, + Final, +} + +impl LostEntry { + fn as_str(self) -> &'static str { + match self { + Self::Chunk => "chunk", + Self::Final => "final", + } + } +} + +/// Report a refusal of `stream`'s `entry` at `pos` on a replica that owes +/// its group a snapshot install. Returns the error the entry's waiter gets. +fn stream_lost_here( + pos: AppliedPosition, + stream: RedoStreamId, + entry: LostEntry, + refusal: &RedoChunkError, +) -> crate::Error { + tracing::error!( + group_id = pos.group_id, + log_index = pos.log_index, + ?stream, + entry = entry.as_str(), + %refusal, + "redo stream: this replica dropped streams of the group when it left it; the entry \ + holds until a snapshot of the group installs" + ); + crate::diag::redo_stream_lost_here( + pos.group_id, + pos.log_index, + stream, + entry.as_str(), + refusal, + ); + crate::Error::Internal { + detail: format!( + "group {}: entry {} of redo stream {stream:?} is held on this replica until a \ + snapshot of the group installs: {refusal}", + pos.group_id, pos.log_index + ), + } +} + +/// Prepare the final entry of a chunked redo: take its stream, assemble the +/// content, and apply it as an inline redo. A stream that is gone or that +/// holds other chunks than the entry names refuses the entry. In a group +/// whose replica here owes a snapshot install, it holds the entry instead. +pub(super) fn prepare_chunked_final<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + target: RedoTarget, + chunked: ChunkedRedo, + incarnations: Vec, +) -> Prepared<'a> { + let taken = ctx.state.redo_chunks.take_for_final(&chunked.stream); + // Every chunk is durable and the final entry committed; nothing of the + // final entry applied yet. + crate::fail_point!("redo_chunk::before_final_install"); + let assembled = match &taken { + Some(open) => open.assemble(chunked.count, chunked.len), + None => Err(RedoChunkError::NotOpen { + stream: chunked.stream, + }), + }; + let bytes = match assembled { + Ok(bytes) => bytes, + Err(refusal) if ctx.state.redo_chunks.owes_snapshot(pos.group_id) => { + let error = stream_lost_here(pos, chunked.stream, LostEntry::Final, &refusal); + return conclude_held(ctx, pos, error, taken); + } + Err(refusal) => { + let result = Err(crate::Error::from(refusal)); + let outcome = EntryOutcome::Applied { + durable: true, + result: Some(ledger_outcome(&result)), + }; + ctx.tracker + .complete(pos.group_id, pos.log_index, pos.applied_key, result); + return Prepared::Concluded(outcome); + } + }; + match RedoContent::from_bytes(&bytes) { + Ok(content) => Prepared::Enqueue(enqueue_payload( + ctx, + pos, + target, + chunked.payload(content), + incarnations, + taken, + )), + Err(error) => conclude_held(ctx, pos, error, taken), + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::{Arc, Mutex}; + + use nodedb_physical::physical_plan::RedoOrigin; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::control::state::SharedState; + use crate::control::wal_replication::decode::{ + DecodedTransactionRedo, decode_transaction_redo, + }; + use crate::control::wal_replication::types::{RedoBody, ReplicatedEventSource}; + use crate::wal::WalManager; + + const GROUP: u64 = 4; + + fn stream() -> RedoStreamId { + RedoStreamId::Session { + vshard: 2, + idempotency_key: 77, + } + } + + fn pos(group_id: u64, log_index: u64) -> AppliedPosition { + AppliedPosition { + group_id, + log_index, + applied_key: 1_000 + log_index, + commit_hlc: 0, + } + } + + fn chunk_entry(index: u32, bytes: &[u8]) -> ReplicatedEntry { + ReplicatedEntry::new( + 1, + 0, + 2, + ReplicatedWrite::RedoChunk { + stream: stream(), + index, + len: 4, + bytes: bytes.to_vec(), + }, + ) + } + + fn final_entry() -> ChunkedRedo { + let write = ReplicatedWrite::TransactionRedo { + body: RedoBody::Chunked { + stream: stream(), + count: 2, + len: 4, + }, + collections: Vec::new(), + event_source: ReplicatedEventSource::User, + origin: RedoOrigin::Commit, + }; + match decode_transaction_redo(&write).expect("decode") { + DecodedTransactionRedo::Chunked(chunked) => chunked, + DecodedTransactionRedo::Inline(_) => panic!("a chunked body decodes chunked"), + } + } + + fn target() -> RedoTarget { + RedoTarget { + tenant_id: TenantId::new(1), + database_id: DatabaseId::DEFAULT, + vshard_id: VShardId::new(2), + } + } + + async fn apply_chunk(ctx: ApplyContext<'_>, pos: AppliedPosition, index: u32) -> EntryOutcome { + let Prepared::Exclusive(apply) = + prepare_stream_entry(ctx, pos, 3, chunk_entry(index, b"ab")) + else { + panic!("a chunk entry applies exclusive"); + }; + apply.await.outcome + } + + fn holds(outcome: &EntryOutcome) -> bool { + matches!( + outcome, + EntryOutcome::Applied { + durable: false, + result: None + } + ) + } + + /// This node applied a stream's first chunk, left the group, and + /// mounted it again on the group's old log. The next chunk and the + /// final entry refuse here while every other replica installs the + /// redo. Each holds the group's durable floor instead of passing. + #[tokio::test] + async fn a_final_entry_of_a_stream_this_replica_dropped_holds_the_apply() { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + let tracker = Arc::new(ProposeTracker::new()); + let senders = Arc::new(Mutex::new(BTreeMap::new())); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + calvin_read_result_senders: &senders, + }; + + let first = apply_chunk(ctx, pos(GROUP, 10), 0).await; + assert!(matches!(first, EntryOutcome::Applied { durable: true, .. })); + state + .redo_chunks + .drop_unhosted_groups(|_| false, state.credentials.catalog()) + .expect("leave the group"); + assert!(state.redo_chunks.owes_snapshot(GROUP)); + + let second = apply_chunk(ctx, pos(GROUP, 11), 1).await; + assert!(holds(&second), "the dropped stream's next chunk holds"); + + let Prepared::Concluded(last) = + prepare_chunked_final(ctx, pos(GROUP, 12), target(), final_entry(), Vec::new()) + else { + panic!("a final entry with no stream concludes at prepare"); + }; + assert!(holds(&last), "the dropped stream's final entry holds"); + } + + /// In a group that owes no snapshot, every replica refuses a final entry + /// with no stream alike, and the refusal passes the entry. + #[tokio::test] + async fn a_final_entry_with_no_stream_passes_in_a_group_that_owes_nothing() { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + let tracker = Arc::new(ProposeTracker::new()); + let senders = Arc::new(Mutex::new(BTreeMap::new())); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + calvin_read_result_senders: &senders, + }; + + let Prepared::Concluded(outcome) = + prepare_chunked_final(ctx, pos(GROUP, 12), target(), final_entry(), Vec::new()) + else { + panic!("a final entry with no stream concludes at prepare"); + }; + assert!(matches!( + outcome, + EntryOutcome::Applied { + durable: true, + result: Some(_) + } + )); + } +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/snapshot_gate.rs b/nodedb/src/control/distributed_applier/apply_loop/snapshot_gate.rs index 32c8e1327..d86ad0bb7 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/snapshot_gate.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/snapshot_gate.rs @@ -30,6 +30,14 @@ pub(super) enum EntryAdmission { Installing, } +/// An entry's admission and the gate state it read. +pub(super) struct Admission { + pub decision: EntryAdmission, + /// The highest snapshot index an install adopted for the group, read + /// under the gate. `0` with no gates, or while an install holds the gate. + pub installed_through: u64, +} + /// Admit entry `log_index` of `group_id`. `gates` is `None` before /// `start_raft` installs them, when no snapshot installs. `covered_through` /// is the cut of the last snapshot this node installed for the group. @@ -38,21 +46,37 @@ pub(super) fn admit_entry( group_id: u64, log_index: u64, covered_through: u64, -) -> EntryAdmission { - let permit = match gates { - None => None, +) -> Admission { + let (permit, installed_through) = match gates { + None => (None, 0), Some(gates) => match gates.try_apply(group_id) { - None => return EntryAdmission::Installing, - Some(permit) if log_index <= permit.installed_through() => { - return EntryAdmission::Covered; + None => { + return Admission { + decision: EntryAdmission::Installing, + installed_through: 0, + }; + } + Some(permit) => { + let installed_through = permit.installed_through(); + if log_index <= installed_through { + return Admission { + decision: EntryAdmission::Covered, + installed_through, + }; + } + (Some(permit), installed_through) } - Some(permit) => Some(permit), }, }; - if log_index <= covered_through { - return EntryAdmission::CoveredByCut; + let decision = if log_index <= covered_through { + EntryAdmission::CoveredByCut + } else { + EntryAdmission::Start(permit) + }; + Admission { + decision, + installed_through, } - EntryAdmission::Start(permit) } /// `work`, holding `permit` until it completes. @@ -79,7 +103,7 @@ mod tests { #[test] fn a_node_without_raft_starts_every_entry() { assert!(matches!( - admit_entry(None, 1, 7, 0), + admit_entry(None, 1, 7, 0).decision, EntryAdmission::Start(None) )); } @@ -91,32 +115,32 @@ mod tests { let gates = Arc::new(GroupApplyGates::new()); gates.mount(1); assert!(matches!( - admit_entry(Some(&gates), 1, 5, 0), + admit_entry(Some(&gates), 1, 5, 0).decision, EntryAdmission::Start(Some(_)) )); let mut install = gates.install(1).await; assert!(matches!( - admit_entry(Some(&gates), 1, 5, 0), + admit_entry(Some(&gates), 1, 5, 0).decision, EntryAdmission::Installing )); install.adopted(5); drop(install); assert!(matches!( - admit_entry(Some(&gates), 1, 4, 0), + admit_entry(Some(&gates), 1, 4, 0).decision, EntryAdmission::Covered )); assert!(matches!( - admit_entry(Some(&gates), 1, 5, 0), + admit_entry(Some(&gates), 1, 5, 0).decision, EntryAdmission::Covered )); assert!(matches!( - admit_entry(Some(&gates), 1, 6, 0), + admit_entry(Some(&gates), 1, 6, 0).decision, EntryAdmission::Start(Some(_)) )); assert!(matches!( - admit_entry(Some(&gates), 2, 5, 0), + admit_entry(Some(&gates), 2, 5, 0).decision, EntryAdmission::Start(Some(_)) )); } @@ -132,34 +156,47 @@ mod tests { gates.install(1).await.adopted(9); let cut = 12; assert!(matches!( - admit_entry(Some(&gates), 1, 9, cut), + admit_entry(Some(&gates), 1, 9, cut).decision, EntryAdmission::Covered )); for log_index in 10..=cut { assert!( matches!( - admit_entry(Some(&gates), 1, log_index, cut), + admit_entry(Some(&gates), 1, log_index, cut).decision, EntryAdmission::CoveredByCut ), "entry {log_index} is in the installed state and must not apply again" ); } assert!(matches!( - admit_entry(Some(&gates), 1, 13, cut), + admit_entry(Some(&gates), 1, 13, cut).decision, EntryAdmission::Start(Some(_)) )); assert!(matches!( - admit_entry(None, 1, 11, cut), + admit_entry(None, 1, 11, cut).decision, EntryAdmission::CoveredByCut )); } + /// Every admission after an install reports the adopted index, so the + /// lane covers it before any later entry starts. + #[tokio::test] + async fn an_admission_reports_the_installed_index() { + let gates = Arc::new(GroupApplyGates::new()); + gates.mount(1); + assert_eq!(admit_entry(Some(&gates), 1, 3, 0).installed_through, 0); + gates.install(1).await.adopted(7); + assert_eq!(admit_entry(Some(&gates), 1, 6, 0).installed_through, 7); + assert_eq!(admit_entry(Some(&gates), 1, 9, 12).installed_through, 7); + assert_eq!(admit_entry(None, 1, 9, 0).installed_through, 0); + } + /// An install cannot start while a started write still holds its permit. #[tokio::test] async fn an_install_waits_for_a_started_write() { let gates = Arc::new(GroupApplyGates::new()); gates.mount(1); - let EntryAdmission::Start(permit) = admit_entry(Some(&gates), 1, 6, 0) else { + let EntryAdmission::Start(permit) = admit_entry(Some(&gates), 1, 6, 0).decision else { panic!("open gate must start the entry"); }; let (release, released) = tokio::sync::oneshot::channel::<()>(); @@ -182,7 +219,7 @@ mod tests { write.await.expect("write task"); install.await.expect("install task"); assert!(matches!( - admit_entry(Some(&gates), 1, 6, 0), + admit_entry(Some(&gates), 1, 6, 0).decision, EntryAdmission::Covered )); } diff --git a/nodedb/src/control/distributed_applier/apply_loop/start.rs b/nodedb/src/control/distributed_applier/apply_loop/start.rs index 87b8f7ff3..7a0fd0089 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/start.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/start.rs @@ -221,6 +221,9 @@ fn prepare_replicated<'a>( ReplicatedWrite::TransactionRedo { .. } => { prepare_transaction_redo_entry(ctx, pos, &replicated, scope.incarnations) } + ReplicatedWrite::RedoChunk { .. } | ReplicatedWrite::RedoAbandon { .. } => { + super::redo_chunk::prepare_stream_entry(ctx, pos, log_term, replicated) + } ReplicatedWrite::SurrogateBind { ref identities } => { Prepared::Concluded(super::surrogate_bind::apply_surrogate_bind( ctx, diff --git a/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs b/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs index da0299849..54b213d74 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs @@ -9,6 +9,9 @@ //! entry re-delivered after a restart is recognised by the proposal ledger and //! skipped before it reaches here. //! +//! A chunked body assembles from the `RedoChunk` entries of its stream +//! (see [`super::redo_chunk`]) and then applies like an inline one. +//! //! A refusal the Data Plane proves applied nothing (a constraint verdict) is //! final: every replica reaches it at the same log position against the same //! state, and the funnel cancels the record in the WAL before it returns. The @@ -20,14 +23,15 @@ use crate::bridge::envelope::Status; use crate::control::array_sync::raft_apply::AppliedPosition; use crate::control::distributed_applier::propose_tracker::{AppliedWrite, ProposeTracker}; use crate::control::server::dispatch_utils::{ChangeFeedOwner, SubmitOutcome, refusal_is_final}; -use crate::control::wal_replication::decode::transaction_redo_payload; +use crate::control::wal_replication::decode::{DecodedTransactionRedo, decode_transaction_redo}; +use crate::control::wal_replication::transaction_redo::chunks::OpenStream; use crate::control::wal_replication::transaction_redo::{ - RedoTarget, enqueue_transaction_redo, record_cross_shard_key, + RedoTarget, TransactionRedoPayload, enqueue_transaction_redo, record_cross_shard_key, }; use crate::control::wal_replication::{CollectionIncarnation, ReplicatedEntry}; use crate::types::{DatabaseId, TenantId, VShardId}; -use super::context::{ApplyContext, FinishedApply, Started, StartedEntry}; +use super::context::{ApplyContext, EnqueueFuture, FinishedApply, Started, StartedEntry}; use super::helpers::committed_response_result; use super::proposal_gate::{EntryOutcome, ledger_outcome}; use super::start::Prepared; @@ -35,7 +39,8 @@ use super::start::Prepared; /// Prepare one committed `TransactionRedo` entry. Its enqueue appends the /// record and hands it to its core. The apply that follows resolves the /// propose waiter. Its outcome says whether the entry's effect is durable on -/// this node, which is what the group's applied prefix records. +/// this node, which is what the group's applied prefix records. A chunked +/// body assembles from its stream first. /// /// `incarnations` are the entry's collection incarnations, moved out of the /// decoded entry. @@ -45,26 +50,74 @@ pub(super) fn prepare_transaction_redo_entry<'a>( entry: &ReplicatedEntry, incarnations: Vec, ) -> Prepared<'a> { - let ApplyContext { state, tracker, .. } = ctx; - let payload = match transaction_redo_payload(&entry.write) { - Ok(payload) => payload, - Err(error) => { - tracker.complete(pos.group_id, pos.log_index, pos.applied_key, Err(error)); - // The entry's own bytes are malformed; a re-delivery decodes the - // same bytes and fails the same way, so it holds the floor rather - // than skipping a committed transaction. - return Prepared::Concluded(EntryOutcome::Applied { - durable: false, - result: None, - }); - } + let decoded = match decode_transaction_redo(&entry.write) { + Ok(decoded) => decoded, + Err(error) => return conclude_held(ctx, pos, error, None), }; let target = RedoTarget { tenant_id: TenantId::new(entry.tenant_id), database_id: DatabaseId::new(entry.database_id), vshard_id: VShardId::new(entry.vshard_id), }; - Prepared::Enqueue(Box::pin(async move { + match decoded { + DecodedTransactionRedo::Inline(payload) => Prepared::Enqueue(enqueue_payload( + ctx, + pos, + target, + *payload, + incarnations, + None, + )), + DecodedTransactionRedo::Chunked(chunked) => { + super::redo_chunk::prepare_chunked_final(ctx, pos, target, chunked, incarnations) + } + } +} + +/// Conclude an entry this replica cannot apply as the other replicas do: +/// its own bytes are malformed, or its stream here is not the one the log +/// built. It holds the floor rather than skipping a committed transaction. +/// A taken stream stays on disk for it. +pub(super) fn conclude_held<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + error: crate::Error, + stream: Option, +) -> Prepared<'a> { + ctx.tracker + .complete(pos.group_id, pos.log_index, pos.applied_key, Err(error)); + let outcome = EntryOutcome::Applied { + durable: false, + result: None, + }; + settle_stream(ctx, stream, &outcome); + Prepared::Concluded(outcome) +} + +/// Release a taken stream once its final entry concluded. A durable outcome +/// drops it. Any other keeps its records on disk, so the next boot rebuilds +/// the stream for the entry's re-apply. +fn settle_stream(ctx: ApplyContext<'_>, stream: Option, outcome: &EntryOutcome) { + let Some(stream) = stream else { + return; + }; + if matches!(outcome, EntryOutcome::Applied { durable: false, .. }) { + ctx.state.redo_chunks.park(stream); + } +} + +/// Route `payload` to its collections, append it, and enqueue it on its +/// core. `stream` is the chunked stream the payload assembled from. +pub(super) fn enqueue_payload<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + target: RedoTarget, + payload: TransactionRedoPayload, + incarnations: Vec, + stream: Option, +) -> EnqueueFuture<'a> { + let ApplyContext { state, tracker, .. } = ctx; + Box::pin(async move { let collection = payload.collections.first().cloned(); // A redo for a collection incarnation this node no longer holds has // nothing to mutate. The gates stay held until the redo is enqueued. @@ -86,17 +139,21 @@ pub(super) fn prepare_transaction_redo_entry<'a>( crate::bridge::envelope::ErrorCode::NotFound, )), ); - return StartedEntry::concluded(EntryOutcome::Applied { + let outcome = EntryOutcome::Applied { durable: true, result: None, - }); + }; + settle_stream(ctx, stream, &outcome); + return StartedEntry::concluded(outcome); } Err(error) => { tracker.complete(pos.group_id, pos.log_index, pos.applied_key, Err(error)); - return StartedEntry::concluded(EntryOutcome::Applied { + let outcome = EntryOutcome::Applied { durable: false, result: None, - }); + }; + settle_stream(ctx, stream, &outcome); + return StartedEntry::concluded(outcome); } }; let enqueued = enqueue_transaction_redo( @@ -120,13 +177,19 @@ pub(super) fn prepare_transaction_redo_entry<'a>( if let Ok(outcome) = &submitted { record_cross_shard_key(state, &payload, outcome); } + let outcome = conclude_transaction_redo(tracker, pos, submitted); + settle_stream(ctx, stream, &outcome); FinishedApply { group_id: pos.group_id, log_index: pos.log_index, - outcome: conclude_transaction_redo(tracker, pos, submitted), + outcome, } })), - Err(error) => Started::Concluded(conclude_transaction_redo(tracker, pos, Err(error))), + Err(error) => { + let outcome = conclude_transaction_redo(tracker, pos, Err(error)); + settle_stream(ctx, stream, &outcome); + Started::Concluded(outcome) + } }; // A committed transaction's redo carries the rows it wrote. StartedEntry { @@ -134,7 +197,7 @@ pub(super) fn prepare_transaction_redo_entry<'a>( collection, user_write: true, } - })) + }) } /// Resolve a transaction redo's waiter from what the funnel returned. diff --git a/nodedb/src/control/gateway/router.rs b/nodedb/src/control/gateway/router.rs index 6df80e2f8..c028f2d6b 100644 --- a/nodedb/src/control/gateway/router.rs +++ b/nodedb/src/control/gateway/router.rs @@ -534,6 +534,7 @@ mod tests { collections: vec![], sum_targets: vec![], origin: nodedb_physical::physical_plan::RedoOrigin::Commit, + calvin: None, }), ] { for table in [None, Some(single_node_table())] { diff --git a/nodedb/src/control/maintenance/clone_materializer/chained.rs b/nodedb/src/control/maintenance/clone_materializer/chained.rs index c4f942e1c..1a3286d82 100644 --- a/nodedb/src/control/maintenance/clone_materializer/chained.rs +++ b/nodedb/src/control/maintenance/clone_materializer/chained.rs @@ -22,9 +22,9 @@ use crate::control::planner::sql_plan_convert::convert::db_qualified; use crate::control::security::catalog::{StoredCollection, SystemCatalog}; use crate::control::state::SharedState; use crate::control::surrogate::CarriedIdentity; -use crate::control::wal_replication::encode::transaction_redo_entry; -use crate::control::wal_replication::propose_replicated_entry; -use crate::control::wal_replication::transaction_redo::{RedoTarget, TransactionRedoPayload}; +use crate::control::wal_replication::transaction_redo::{ + RedoTarget, TransactionRedoPayload, propose_transaction_redo, +}; use crate::event::EventSource; use crate::types::hash_chain::CHAIN_SEQ_FIELD; use crate::types::{HomedRecord, RecordHomes}; @@ -249,6 +249,7 @@ fn relink_payload( // a copy fires no AFTER trigger. event_source: EventSource::Restore, origin: RedoOrigin::Restore, + calvin: None, }) } @@ -258,14 +259,7 @@ async fn commit_relink( target: RedoTarget, payload: &TransactionRedoPayload, ) -> crate::Result<()> { - let proposer = state.async_raft_proposer()?; - let entry = transaction_redo_entry( - target.tenant_id, - target.database_id, - target.vshard_id, - payload, - ); let deadline = crate::control::wal_replication::statement_propose_deadline(state); - propose_replicated_entry(state, proposer, entry, deadline).await?; + propose_transaction_redo(state, target, payload, deadline).await?; Ok(()) } diff --git a/nodedb/src/control/metadata_proposer/catalog.rs b/nodedb/src/control/metadata_proposer/catalog.rs index f964a878a..d71f2408c 100644 --- a/nodedb/src/control/metadata_proposer/catalog.rs +++ b/nodedb/src/control/metadata_proposer/catalog.rs @@ -233,7 +233,7 @@ pub(super) async fn propose_prepared( .await?; match outcome { WaitOutcome::Reached - if shared.metadata_ddl_applied_token.load(Ordering::Acquire) == token => + if shared.metadata_ddl.applied_token.load(Ordering::Acquire) == token => { Ok(log_index) } diff --git a/nodedb/src/control/metadata_proposer/ddl_owner.rs b/nodedb/src/control/metadata_proposer/ddl_owner.rs index 70de3f1a2..9c4c83e57 100644 --- a/nodedb/src/control/metadata_proposer/ddl_owner.rs +++ b/nodedb/src/control/metadata_proposer/ddl_owner.rs @@ -80,7 +80,8 @@ pub(crate) fn reclaim_cause(shared: &SharedState, owner: &DdlPrepareOwner) -> Op /// Whether `token` owns the preparation lease on this replica. pub(crate) fn owns_ddl_lease(shared: &SharedState, token: u64) -> bool { shared - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .unwrap_or_else(|p| p.into_inner()) .is_some_and(|owner| owner.token == token) @@ -89,7 +90,8 @@ pub(crate) fn owns_ddl_lease(shared: &SharedState, token: u64) -> bool { /// The current owner on this replica, if the lease is held. pub(crate) fn current_owner(shared: &SharedState) -> Option { *shared - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .unwrap_or_else(|p| p.into_inner()) } diff --git a/nodedb/src/control/metadata_proposer/ddl_prepare.rs b/nodedb/src/control/metadata_proposer/ddl_prepare.rs index 9fefdee02..2e03fcc77 100644 --- a/nodedb/src/control/metadata_proposer/ddl_prepare.rs +++ b/nodedb/src/control/metadata_proposer/ddl_prepare.rs @@ -35,7 +35,8 @@ const DDL_PREPARE_POLL: Duration = Duration::from_millis(10); /// A fresh preparation-lease token for this node. fn next_token(shared: &SharedState) -> u64 { let sequence = shared - .metadata_ddl_token_seq + .metadata_ddl + .token_seq .fetch_add(1, Ordering::Relaxed); shared.node_id.wrapping_mul(0x9e37_79b9_7f4a_7c15) ^ wall_now_ns().rotate_left(17) ^ sequence } @@ -214,7 +215,7 @@ pub(crate) async fn acquire_ddl_prepare_lease_async<'a>( pub(crate) async fn lock_ddl_preparation_async( shared: &SharedState, ) -> tokio::sync::MutexGuard<'_, ()> { - shared.metadata_ddl_lock.lock().await + shared.metadata_ddl.lock.lock().await } #[cfg(test)] @@ -223,7 +224,8 @@ mod tests { fn owner(shared: &SharedState) -> Option { let current = *shared - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .unwrap_or_else(|p| p.into_inner()); current.map(|owner| owner.token) diff --git a/nodedb/src/control/security/auth_fence/state.rs b/nodedb/src/control/security/auth_fence/state.rs index fd010136b..51995c51d 100644 --- a/nodedb/src/control/security/auth_fence/state.rs +++ b/nodedb/src/control/security/auth_fence/state.rs @@ -8,7 +8,6 @@ use std::sync::{Arc, Mutex, OnceLock}; use tokio::sync::Notify; -use crate::control::cluster::calvin::scheduler::AppliedMirrors; use crate::control::security::auth_lease::{ CalvinAckCoverage, LeaderLeaseService, LeaseHolder, LeaseTiming, }; @@ -31,8 +30,6 @@ pub struct AuthorizationFence { tree_defs: PendingTreeDefs, /// The permission cache's source collections, readable without its lock. sources: Arc, - /// Which Calvin positions this node's schedulers applied. - calvin_mirrors: AppliedMirrors, /// Sequencer completion acks not yet settled against local schedulers. calvin_acks: CalvinAckCoverage, /// This node's authorization lease. @@ -56,7 +53,6 @@ impl AuthorizationFence { read_index: Mutex::new(HashMap::new()), tree_defs: PendingTreeDefs::default(), sources, - calvin_mirrors: AppliedMirrors::default(), calvin_acks: CalvinAckCoverage::default(), holder: LeaseHolder::default(), timing: OnceLock::new(), @@ -106,11 +102,6 @@ impl AuthorizationFence { &self.sources } - /// Which Calvin positions this node's schedulers applied. - pub fn calvin_mirrors(&self) -> &AppliedMirrors { - &self.calvin_mirrors - } - /// Sequencer completion acks not yet settled against local schedulers. pub fn calvin_acks(&self) -> &CalvinAckCoverage { &self.calvin_acks diff --git a/nodedb/src/control/security/auth_lease/calvin_acks.rs b/nodedb/src/control/security/auth_lease/calvin_acks.rs index 3d6ed6362..26b8b52fe 100644 --- a/nodedb/src/control/security/auth_lease/calvin_acks.rs +++ b/nodedb/src/control/security/auth_lease/calvin_acks.rs @@ -10,15 +10,15 @@ //! its own scheduler applied that transaction too. //! //! The sequencer state machine records each ack it applies. This tracker -//! takes them in log order and settles each against the local scheduler's -//! applied mirror. The first unsettled ack bounds the coverage. +//! takes them in log order and settles each against the vShard's applied +//! ledger on this node. The first unsettled ack bounds the coverage. use std::collections::VecDeque; use std::sync::Mutex; use nodedb_cluster::calvin::{AppliedCompletionAck, CalvinCompletionRegistry}; -use crate::control::cluster::calvin::scheduler::AppliedMirrors; +use crate::control::cluster::calvin::scheduler::CalvinAppliedLedgers; /// Acks taken from the sequencer log and not yet settled. #[derive(Debug, Default)] @@ -36,7 +36,7 @@ impl CalvinAckCoverage { pub fn covered_through( &self, registry: &CalvinCompletionRegistry, - mirrors: &AppliedMirrors, + ledgers: &CalvinAppliedLedgers, applied: u64, replicates: impl Fn(u32) -> bool, ) -> u64 { @@ -44,9 +44,9 @@ impl CalvinAckCoverage { pending.extend(registry.applied_acks.drain()); while let Some(ack) = pending.front() { let settled = !replicates(ack.vshard_id) - || mirrors + || ledgers .get(ack.vshard_id) - .is_some_and(|mirror| mirror.is_applied(ack.txn.epoch, ack.txn.position)); + .is_some_and(|ledger| ledger.is_applied(ack.txn.epoch, ack.txn.position)); if !settled { break; } @@ -80,8 +80,8 @@ mod tests { fn an_ack_the_local_replica_has_not_applied_bounds_the_coverage() { let registry = CalvinCompletionRegistry::new_detached(); registry.applied_acks.enable(); - let mirrors = AppliedMirrors::default(); - let mirror = mirrors.register(7, NOT_YET_APPLIED_EPOCH, &BTreeSet::new()); + let ledgers = CalvinAppliedLedgers::default(); + let ledger = ledgers.install(7, NOT_YET_APPLIED_EPOCH, BTreeSet::new()); let coverage = CalvinAckCoverage::default(); registry.applied_acks.record(ack(4, 1, 7)); @@ -89,38 +89,38 @@ mod tests { // vShard 7 is replicated here and its scheduler has not applied epoch 1. let replicates = |vshard: u32| vshard == 7; assert_eq!( - coverage.covered_through(®istry, &mirrors, 8, replicates), + coverage.covered_through(®istry, &ledgers, 8, replicates), 3 ); - mirror.mark(1, 0); + ledger.mark_applied(1, 0); // The ack for vShard 9 settles at once: it is not replicated here. assert_eq!( - coverage.covered_through(®istry, &mirrors, 8, replicates), + coverage.covered_through(®istry, &ledgers, 8, replicates), 8 ); } #[test] - fn a_replicated_vshard_without_a_scheduler_yet_is_unsettled() { + fn a_replicated_vshard_without_a_ledger_is_unsettled() { let registry = CalvinCompletionRegistry::new_detached(); registry.applied_acks.enable(); - let mirrors = AppliedMirrors::default(); + let ledgers = CalvinAppliedLedgers::default(); let coverage = CalvinAckCoverage::default(); registry.applied_acks.record(ack(2, 1, 5)); assert_eq!( - coverage.covered_through(®istry, &mirrors, 3, |_| true), + coverage.covered_through(®istry, &ledgers, 3, |_| true), 1 ); } - /// A restart loses the in-memory mirror and, after a checkpoint, the WAL - /// markers of transactions applied before it. The mirror rebuilt from the + /// A restart loses the in-memory ledger and, after a checkpoint, the WAL + /// markers of transactions applied before it. The ledger rebuilt from the /// state the checkpoint saved settles every ack the sequencer log replays, /// so the coverage reaches the sequencer log's applied index. #[test] - fn a_mirror_rebuilt_from_saved_state_settles_replayed_acks() { - use crate::control::cluster::calvin::scheduler::recover_applied; + fn a_ledger_rebuilt_from_saved_state_settles_replayed_acks() { + use crate::control::cluster::calvin::scheduler::recover_all_applied; use crate::control::security::catalog::SystemCatalog; use crate::control::security::catalog::calvin_applied::StoredCalvinApplied; @@ -138,9 +138,10 @@ mod tests { }]) .expect("save"); - let recovered = recover_applied(&wal, &catalog, 7, None).expect("recover"); - let mirrors = AppliedMirrors::default(); - mirrors.register(7, recovered.fully_applied_epoch, &recovered.applied_tail); + let mut recovered = recover_all_applied(&wal, &catalog, &|_| None).expect("recover"); + let recovered = recovered.remove(&7).expect("vShard 7 recovered"); + let ledgers = CalvinAppliedLedgers::default(); + ledgers.install(7, recovered.fully_applied_epoch, recovered.applied_tail); let registry = CalvinCompletionRegistry::new_detached(); registry.applied_acks.enable(); @@ -156,9 +157,9 @@ mod tests { }); let coverage = CalvinAckCoverage::default(); assert_eq!( - coverage.covered_through(®istry, &mirrors, 19, |_| true), + coverage.covered_through(®istry, &ledgers, 19, |_| true), 19, - "every replayed ack settles against the rebuilt mirror" + "every replayed ack settles against the rebuilt ledger" ); } } diff --git a/nodedb/src/control/security/auth_lease/coverage.rs b/nodedb/src/control/security/auth_lease/coverage.rs index e0836b587..3722dedbf 100644 --- a/nodedb/src/control/security/auth_lease/coverage.rs +++ b/nodedb/src/control/security/auth_lease/coverage.rs @@ -78,7 +78,7 @@ fn sequencer_coverage(state: &SharedState) -> u64 { let fence = &state.authorization_fence; fence .calvin_acks() - .covered_through(registry, fence.calvin_mirrors(), applied, |vshard_id| { + .covered_through(registry, &state.calvin.applied, applied, |vshard_id| { group_of_vshard(state, vshard_id).is_ok_and(|g| hosts_group(state, g)) }) } diff --git a/nodedb/src/control/security/catalog/bootstrap_tables.rs b/nodedb/src/control/security/catalog/bootstrap_tables.rs index 35b42b2f8..a99c5290d 100644 --- a/nodedb/src/control/security/catalog/bootstrap_tables.rs +++ b/nodedb/src/control/security/catalog/bootstrap_tables.rs @@ -81,6 +81,7 @@ pub(super) const BOOTSTRAP_TABLES: &[BootstrapTable] = bootstrap_tables![ "calvin_applied" => super::calvin_applied::CALVIN_APPLIED, "calvin_base" => super::calvin_base::CALVIN_BASE, "calvin_sequencer_install" => super::calvin_base::CALVIN_SEQUENCER_INSTALL, + "redo_snapshot_owed" => super::redo_snapshot_owed::REDO_SNAPSHOT_OWED, "l2_cleanup_queue" => L2_CLEANUP_QUEUE, "pending_reclaim" => PENDING_RECLAIM, // ── Metadata-group host state ── diff --git a/nodedb/src/control/security/catalog/calvin_applied.rs b/nodedb/src/control/security/catalog/calvin_applied.rs index 18ca14c20..e003c87c0 100644 --- a/nodedb/src/control/security/catalog/calvin_applied.rs +++ b/nodedb/src/control/security/catalog/calvin_applied.rs @@ -75,30 +75,29 @@ fn decode_tail(bytes: &[u8]) -> crate::Result> { } impl SystemCatalog { - /// The saved applied state of `vshard_id`, if any. - pub fn load_calvin_applied( - &self, - vshard_id: u32, - ) -> crate::Result> { + /// The saved applied state of every vShard. + pub fn load_all_calvin_applied(&self) -> crate::Result> { let read_txn = self .db .begin_read() - .map_err(|e| catalog_err("load_calvin_applied read txn", e))?; + .map_err(|e| catalog_err("load_all_calvin_applied read txn", e))?; let table = read_txn .open_table(CALVIN_APPLIED) .map_err(|e| catalog_err("open calvin_applied", e))?; - let Some(row) = table - .get(vshard_id) - .map_err(|e| catalog_err("get calvin_applied", e))? - else { - return Ok(None); - }; - let (fully_applied_epoch, tail) = row.value(); - Ok(Some(StoredCalvinApplied { - vshard_id, - fully_applied_epoch, - tail: decode_tail(tail)?, - })) + let mut out = Vec::new(); + for row in table + .iter() + .map_err(|e| catalog_err("iter calvin_applied", e))? + { + let (vshard, value) = row.map_err(|e| catalog_err("read calvin_applied", e))?; + let (fully_applied_epoch, tail) = value.value(); + out.push(StoredCalvinApplied { + vshard_id: vshard.value(), + fully_applied_epoch, + tail: decode_tail(tail)?, + }); + } + Ok(out) } /// Replace the saved state of each vShard in `states` with the given one. @@ -198,7 +197,7 @@ mod tests { fn saved_state_reloads_and_only_grows() { let dir = tempfile::tempdir().expect("tempdir"); let catalog = SystemCatalog::open(&dir.path().join("system.redb")).expect("catalog"); - assert_eq!(catalog.load_calvin_applied(3).expect("load"), None); + assert!(catalog.load_all_calvin_applied().expect("load").is_empty()); catalog .save_calvin_applied(vec![state(3, NONE_FULLY_APPLIED, &[(0, 0), (2, 1)])]) @@ -207,8 +206,8 @@ mod tests { .save_calvin_applied(vec![state(3, 1, &[(4, 0)])]) .expect("save"); assert_eq!( - catalog.load_calvin_applied(3).expect("load"), - Some(state(3, 1, &[(2, 1), (4, 0)])) + catalog.load_all_calvin_applied().expect("load"), + vec![state(3, 1, &[(2, 1), (4, 0)])] ); } } diff --git a/nodedb/src/control/security/catalog/mod.rs b/nodedb/src/control/security/catalog/mod.rs index 0f37cebd1..66b8de550 100644 --- a/nodedb/src/control/security/catalog/mod.rs +++ b/nodedb/src/control/security/catalog/mod.rs @@ -59,6 +59,7 @@ pub mod procedure_types; pub mod procedures; pub mod read_only; pub mod redaction; +pub mod redo_snapshot_owed; pub mod replicated_image; pub mod replicated_image_merge; pub mod restore_points; diff --git a/nodedb/src/control/security/catalog/redo_snapshot_owed.rs b/nodedb/src/control/security/catalog/redo_snapshot_owed.rs new file mode 100644 index 000000000..6ae86066a --- /dev/null +++ b/nodedb/src/control/security/catalog/redo_snapshot_owed.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The data groups whose replica on this node owes a snapshot install, +//! backing `_system.redo_snapshot_owed`. +//! +//! A node that leaves a data group drops the group's open chunked redo +//! streams. A later remount resumes the group's old log, which can hold a +//! later chunk or final entry of a dropped stream. The row makes the remount +//! catch up by snapshot instead. A snapshot install of the group removes it. +//! +//! Table: `group_id` -> `()`. + +use redb::{ReadableDatabase, ReadableTable, TableDefinition}; + +use super::types::{SystemCatalog, catalog_err}; + +pub(super) const REDO_SNAPSHOT_OWED: TableDefinition = + TableDefinition::new("_system.redo_snapshot_owed"); + +impl SystemCatalog { + /// Every data group whose replica here owes a snapshot install. + pub fn load_redo_snapshot_owed(&self) -> crate::Result> { + let txn = self + .db + .begin_read() + .map_err(|e| catalog_err("redo_snapshot_owed read txn", e))?; + let table = txn + .open_table(REDO_SNAPSHOT_OWED) + .map_err(|e| catalog_err("open redo_snapshot_owed", e))?; + let mut out = Vec::new(); + for row in table + .iter() + .map_err(|e| catalog_err("iterate redo_snapshot_owed", e))? + { + let (group_id, _) = row.map_err(|e| catalog_err("read redo_snapshot_owed", e))?; + out.push(group_id.value()); + } + Ok(out) + } + + /// Record in one transaction that the replicas of `groups` here owe a + /// snapshot install. + pub fn save_redo_snapshot_owed(&self, groups: &[u64]) -> crate::Result<()> { + if groups.is_empty() { + return Ok(()); + } + let txn = self + .db + .begin_write() + .map_err(|e| catalog_err("redo_snapshot_owed write txn", e))?; + { + let mut table = txn + .open_table(REDO_SNAPSHOT_OWED) + .map_err(|e| catalog_err("open redo_snapshot_owed", e))?; + for &group_id in groups { + table + .insert(group_id, ()) + .map_err(|e| catalog_err("insert redo_snapshot_owed", e))?; + } + } + txn.commit() + .map_err(|e| catalog_err("redo_snapshot_owed commit", e)) + } + + /// Remove the row of `group_id`. Removing an absent row succeeds. + pub fn remove_redo_snapshot_owed(&self, group_id: u64) -> crate::Result<()> { + let txn = self + .db + .begin_write() + .map_err(|e| catalog_err("redo_snapshot_owed write txn", e))?; + { + let mut table = txn + .open_table(REDO_SNAPSHOT_OWED) + .map_err(|e| catalog_err("open redo_snapshot_owed", e))?; + table + .remove(group_id) + .map_err(|e| catalog_err("remove redo_snapshot_owed", e))?; + } + txn.commit() + .map_err(|e| catalog_err("redo_snapshot_owed commit", e)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn owed_groups_save_reload_and_remove() { + let dir = tempfile::tempdir().expect("tempdir"); + let catalog = SystemCatalog::open(&dir.path().join("system.redb")).expect("catalog"); + assert!(catalog.load_redo_snapshot_owed().expect("load").is_empty()); + + catalog.save_redo_snapshot_owed(&[4, 9]).expect("save"); + catalog.save_redo_snapshot_owed(&[4]).expect("save again"); + assert_eq!(catalog.load_redo_snapshot_owed().expect("load"), vec![4, 9]); + + catalog.remove_redo_snapshot_owed(4).expect("remove"); + catalog + .remove_redo_snapshot_owed(5) + .expect("remove an absent row"); + assert_eq!(catalog.load_redo_snapshot_owed().expect("load"), vec![9]); + } +} diff --git a/nodedb/src/control/security/catalog/replicated_image.rs b/nodedb/src/control/security/catalog/replicated_image.rs index c59552a6f..8dadcd828 100644 --- a/nodedb/src/control/security/catalog/replicated_image.rs +++ b/nodedb/src/control/security/catalog/replicated_image.rs @@ -23,6 +23,8 @@ //! - `tenant_group_marks`, `tenant_group_restore_marks`: data-group write //! marks, carried by data-group snapshots. //! - `calvin_applied`: this node's Calvin apply ledger. +//! - `redo_snapshot_owed`: the data-group snapshot installs this node's +//! replicas owe. //! - `l2_cleanup_queue`, `pending_reclaim`, `pending_history_compaction`: //! work this node still owes its own storage. //! - `surrogate_pk_v3`, `surrogate_pk_rev_v3`: PK bindings of stored rows, diff --git a/nodedb/src/control/server/dispatch_utils/durability_barrier.rs b/nodedb/src/control/server/dispatch_utils/durability_barrier.rs index 699eccf1e..6f5a94006 100644 --- a/nodedb/src/control/server/dispatch_utils/durability_barrier.rs +++ b/nodedb/src/control/server/dispatch_utils/durability_barrier.rs @@ -169,6 +169,7 @@ mod tests { collections: vec!["c".into()], sum_targets: Vec::new(), origin: nodedb_physical::physical_plan::RedoOrigin::Commit, + calvin: None, }); assert_eq!(funnel_minted_redo_engine(&plan), Some("transaction")); } diff --git a/nodedb/src/control/server/exchange/all_cores/snapshot.rs b/nodedb/src/control/server/exchange/all_cores/snapshot.rs index 9c54e153c..90ecab08f 100644 --- a/nodedb/src/control/server/exchange/all_cores/snapshot.rs +++ b/nodedb/src/control/server/exchange/all_cores/snapshot.rs @@ -135,6 +135,7 @@ pub(super) fn merge_core_snapshots( // cut on the merged payload. A per-core part carries neither. group_event_lane: _, group_calvin: _, + group_redo_streams: _, } = part; // The group snapshot builder sets the cut on the merged payload. A // per-core part carries none. diff --git a/nodedb/src/control/server/ilp_batch/dispatch.rs b/nodedb/src/control/server/ilp_batch/dispatch.rs index 877d1fba4..b79363828 100644 --- a/nodedb/src/control/server/ilp_batch/dispatch.rs +++ b/nodedb/src/control/server/ilp_batch/dispatch.rs @@ -96,7 +96,7 @@ pub(crate) async fn flush_authenticated_ilp_batch( /// At most one schema-projection merge is ever in flight, process-wide. /// /// The merge is a replicated catalog DDL, and every catalog DDL already -/// serializes on `SharedState::metadata_ddl_lock`. A second concurrent merge +/// serializes on `MetadataDdlState::lock`. A second concurrent merge /// can therefore only park a second task on a lock that admits one holder, /// so one permit is both the useful and the safe bound — ingest can never /// queue merges no matter how many ILP connections or OTLP requests are live. diff --git a/nodedb/src/control/server/shared/session/commit/single_shard.rs b/nodedb/src/control/server/shared/session/commit/single_shard.rs index 6d3fee1d1..87fa6cee8 100644 --- a/nodedb/src/control/server/shared/session/commit/single_shard.rs +++ b/nodedb/src/control/server/shared/session/commit/single_shard.rs @@ -14,9 +14,9 @@ use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; use crate::bridge::envelope::{PhysicalPlan, Status}; use crate::control::gateway::RouteDecision; use crate::control::state::SharedState; -use crate::control::wal_replication::encode::transaction_redo_entry; -use crate::control::wal_replication::propose_replicated_entry; -use crate::control::wal_replication::transaction_redo::{RedoTarget, TransactionRedoPayload}; +use crate::control::wal_replication::transaction_redo::{ + RedoTarget, TransactionRedoPayload, propose_transaction_redo, +}; use crate::wal::RedoPublish; use super::super::outcome::{AbortReason, TxnDataPlane}; @@ -204,17 +204,11 @@ async fn commit_redo( if let Err(e) = inject_commit_failure() { return Err(AbortReason::Dispatch(e)); } - let proposer = state.async_raft_proposer().map_err(AbortReason::Dispatch)?; - // The proposer forwards to the group leader and returns once the entry is - // committed and applied on this node by the apply loop. - let entry = transaction_redo_entry( - target.tenant_id, - target.database_id, - target.vshard_id, - payload, - ); + // The proposal forwards to the group leader and returns once the redo is + // committed and applied on this node by the apply loop. A redo past the + // entry limit travels as chunks, then its final entry. let deadline = crate::control::wal_replication::statement_propose_deadline(state); - match propose_replicated_entry(state, proposer, entry, deadline).await { + match propose_transaction_redo(state, target, payload, deadline).await { Ok((applied, _)) => Ok(super::ts_rejections::applied_rejected_by_collection( &applied, )), diff --git a/nodedb/src/control/server/shared/session/ddl_compensate.rs b/nodedb/src/control/server/shared/session/ddl_compensate.rs index 651de26d5..e698bc197 100644 --- a/nodedb/src/control/server/shared/session/ddl_compensate.rs +++ b/nodedb/src/control/server/shared/session/ddl_compensate.rs @@ -73,7 +73,7 @@ async fn propose_reversals( entry: Box::new(MetadataEntry::Batch { entries }), }; let log_index = propose_and_await(state, handle, &prepared).await?; - if state.metadata_ddl_applied_token.load(Ordering::Acquire) != token { + if state.metadata_ddl.applied_token.load(Ordering::Acquire) != token { return Err(crate::Error::Config { detail: "commit compensation: DDL preparation ownership was superseded before apply" .into(), diff --git a/nodedb/src/control/server/shared/session/ddl_flush.rs b/nodedb/src/control/server/shared/session/ddl_flush.rs index 3499474d7..296a862cf 100644 --- a/nodedb/src/control/server/shared/session/ddl_flush.rs +++ b/nodedb/src/control/server/shared/session/ddl_flush.rs @@ -373,7 +373,8 @@ async fn finalize_reserved( // cancels the record first, so a finalize that found nothing applied // nothing. if state - .metadata_ddl_applied_token + .metadata_ddl + .applied_token .load(std::sync::atomic::Ordering::Acquire) != token { @@ -713,7 +714,8 @@ mod tests { ); assert!( state - .metadata_ddl_owner + .metadata_ddl + .owner .lock() .expect("owner lock") .is_none(), diff --git a/nodedb/src/control/server/shared/write_admission/leader_gate.rs b/nodedb/src/control/server/shared/write_admission/leader_gate.rs index b3eccc32f..094b8bf09 100644 --- a/nodedb/src/control/server/shared/write_admission/leader_gate.rs +++ b/nodedb/src/control/server/shared/write_admission/leader_gate.rs @@ -61,7 +61,12 @@ impl DataProposeGate for LeaderWriteGate { return Ok(None); }; let vshard = VShardId::new(vshard_id); - let guard = match admit_entry(&state, vshard, entry).map_err(gate_error)? { + // Bytes that are not a replicated entry apply as nothing on any replica. + let Some(entry) = ReplicatedEntry::from_bytes(entry) else { + return Ok(None); + }; + admit_redo_stream(&state, &entry, deadline).map_err(gate_error)?; + let guard = match admit_entry(&state, vshard, &entry).map_err(gate_error)? { WriteAdmission::ExemptRead | WriteAdmission::FastPath { guard: None } // No scheduler runs for the vShard here: the log alone orders @@ -112,22 +117,41 @@ enum EntryScope { Plan, } -/// Admit the encoded entry `entry`, proposed to `vshard`'s group. +/// Admit the first chunk of a chunked redo stream against this node's open +/// stream bytes. Every replica takes the chunks the log holds, so the leader +/// is the one place a stream is refused for memory. Any other entry passes. +fn admit_redo_stream( + state: &SharedState, + entry: &ReplicatedEntry, + deadline: tokio::time::Instant, +) -> crate::Result<()> { + let ReplicatedWrite::RedoChunk { + stream, + index: 0, + len, + .. + } = &entry.write + else { + return Ok(()); + }; + state + .redo_chunks + .admit_stream(*stream, *len, deadline.into_std()) + .map_err(crate::Error::from) +} + +/// Admit `entry`, proposed to `vshard`'s group. pub(crate) fn admit_entry( state: &SharedState, vshard: VShardId, - entry: &[u8], + entry: &ReplicatedEntry, ) -> crate::Result { - // Bytes that are not a replicated entry apply as nothing on any replica. - let Some(entry) = ReplicatedEntry::from_bytes(entry) else { - return Ok(WriteAdmission::ExemptRead); - }; let database_id = crate::types::DatabaseId::new(entry.database_id); match entry_scope(&entry.write, database_id) { EntryScope::Exempt => Ok(WriteAdmission::ExemptRead), EntryScope::Collections(collections) => Ok(admit_collections(state, vshard, &collections)), EntryScope::Plan => { - let Some((_, (tenant_id, _, plan, _))) = decode_parsed_entry(&entry)? else { + let Some((_, (tenant_id, _, plan, _))) = decode_parsed_entry(entry)? else { return Ok(WriteAdmission::ExemptRead); }; // An entry the scheduler cannot apply as proposed waits for its @@ -165,11 +189,19 @@ fn entry_scope(write: &ReplicatedWrite, database_id: crate::types::DatabaseId) - // scheduler holds its locks. A session transaction's redo writes // every collection it names. ReplicatedWrite::TransactionRedo { - redo, collections, .. - } => match redo.calvin_stamp { - Some(_) => EntryScope::Exempt, - None => EntryScope::Collections(collections.clone()), - }, + body, collections, .. + } => { + if body.is_calvin() { + EntryScope::Exempt + } else { + EntryScope::Collections(collections.clone()) + } + } + // A redo chunk and an abandon write no row: the stream's final entry + // takes the locks. + ReplicatedWrite::RedoChunk { .. } | ReplicatedWrite::RedoAbandon { .. } => { + EntryScope::Exempt + } ReplicatedWrite::ArrayOp { array, .. } => EntryScope::Collections(vec![ QualifiedCollection::new(database_id, array) .as_str() diff --git a/nodedb/src/control/server/wal_dispatch/core.rs b/nodedb/src/control/server/wal_dispatch/core.rs index b7ef91567..bb54d0d86 100644 --- a/nodedb/src/control/server/wal_dispatch/core.rs +++ b/nodedb/src/control/server/wal_dispatch/core.rs @@ -259,6 +259,7 @@ mod tests { collections: Vec::new(), sum_targets: Vec::new(), origin: nodedb_physical::physical_plan::RedoOrigin::Commit, + calvin: None, }, ); diff --git a/nodedb/src/control/state/calvin_local.rs b/nodedb/src/control/state/calvin_local.rs index 974363ad5..723f7b137 100644 --- a/nodedb/src/control/state/calvin_local.rs +++ b/nodedb/src/control/state/calvin_local.rs @@ -7,9 +7,11 @@ use std::collections::BTreeMap; use std::sync::atomic::{AtomicU32, AtomicU64}; use std::sync::{Arc, Mutex, OnceLock}; -use crate::control::cluster::calvin::scheduler::SequencerProposer; use crate::control::cluster::calvin::scheduler::lock::HotKeyTable; use crate::control::cluster::calvin::scheduler::lock_manager::{LockManager, TxnId}; +use crate::control::cluster::calvin::scheduler::{ + CalvinAppliedLedgers, CaughtUpRegistry, SequencerProposer, +}; use crate::control::server::shared::write_admission::AdmissionHolds; use super::calvin_apply_sidecar::CalvinApplySidecar; @@ -84,6 +86,12 @@ pub struct CalvinLocalState { /// Hands this node's sequencer entries to the sequencer Raft group. Set /// once the schedulers start; unset on a node that runs none. pub sequencer_proposer: OnceLock>, + /// Which Calvin positions this node's replica of each vShard applied. + /// Boot recovery fills it before the data-group apply loop starts. + pub applied: CalvinAppliedLedgers, + /// Whether each running scheduler reached its rebuild target. Startup + /// reads it before it opens the client gateway. + pub caught_up: CaughtUpRegistry, } impl CalvinLocalState { @@ -106,6 +114,8 @@ impl CalvinLocalState { cuts: CalvinCuts::default(), bases: CalvinBases::default(), sequencer_proposer: OnceLock::new(), + applied: CalvinAppliedLedgers::default(), + caught_up: CaughtUpRegistry::default(), } } } diff --git a/nodedb/src/control/state/fields.rs b/nodedb/src/control/state/fields.rs index d5a6327cd..f1cbdb6e2 100644 --- a/nodedb/src/control/state/fields.rs +++ b/nodedb/src/control/state/fields.rs @@ -117,23 +117,12 @@ pub struct SharedState { >, /// Per-Raft-group apply watermark registry for commit-wait and drain paths. pub group_watchers: Arc, - /// Serializes this node's attempts to acquire the replicated descriptor - /// preparation lease, and a local DDL through its post-apply. An async - /// proposer holds it across the post-apply, so it is a tokio mutex. - pub metadata_ddl_lock: tokio::sync::Mutex<()>, - /// Replicated preparation owner: its token, its node, and the local - /// monotonic apply time. The token and node are persisted in - /// `SystemCatalog` and seeded at boot. - pub metadata_ddl_owner: Mutex>, - /// Most recent fenced DDL token applied while its owner remained current. - /// Starts at 0 on boot. - pub metadata_ddl_applied_token: AtomicU64, + /// This node's state of the replicated descriptor preparation lease. + pub metadata_ddl: super::metadata_ddl::MetadataDdlState, /// Apply-progress steps this node made on metadata entries: each batch /// sub-entry and each purge-reclaim step (a core acknowledgement, a closed /// scan). A proposer on a long apply reads it to tell slow from stalled. pub metadata_apply_progress: AtomicU64, - /// Per-node uniqueness component for descriptor-preparation lease tokens. - pub metadata_ddl_token_seq: AtomicU64, /// Node-local table of in-flight `DdlPendingPropose` records. Persisted in /// `SystemCatalog` and seeded at boot. pub pending_ddl: crate::control::pending_ddl::PendingDdlTable, @@ -294,6 +283,9 @@ pub struct SharedState { /// setup. Every committed redo that carries a cross-shard key records it /// here as it applies. pub cross_shard_dedup: OnceLock>, + /// Open chunked redo streams: the chunks every data-group apply holds + /// until each stream's final entry. + pub redo_chunks: Arc, /// Kafka bridge producer manager. pub kafka_manager: crate::event::kafka::KafkaManager, /// Definition sync fanout: broadcasts `DefinitionSync` (0x70) frames to diff --git a/nodedb/src/control/state/init.rs b/nodedb/src/control/state/init.rs index 540a0291f..006096ab6 100644 --- a/nodedb/src/control/state/init.rs +++ b/nodedb/src/control/state/init.rs @@ -106,6 +106,14 @@ impl SharedState { // as the production constructor wires it. let stream_registry = Arc::new(crate::event::cdc::StreamRegistry::new()); + let redo_chunks = Arc::new( + crate::control::wal_replication::transaction_redo::RedoChunkStore::new( + Arc::clone(&wal), + crate::control::wal_replication::transaction_redo::chunks::RedoChunkLimits::from_tuning( + &TuningConfig::default().calvin, + ), + ), + ); let hlc_clock = wal.hlc_clock(); let state = Arc::new(Self { outcome_floor: dispatcher.outcome_floor(), @@ -130,11 +138,8 @@ impl SharedState { ) .0, group_watchers: Arc::new(nodedb_cluster::GroupAppliedWatchers::new()), - metadata_ddl_lock: tokio::sync::Mutex::new(()), - metadata_ddl_owner: std::sync::Mutex::new(None), - metadata_ddl_applied_token: std::sync::atomic::AtomicU64::new(0), + metadata_ddl: super::MetadataDdlState::new(), metadata_apply_progress: std::sync::atomic::AtomicU64::new(0), - metadata_ddl_token_seq: std::sync::atomic::AtomicU64::new(1), pending_ddl: crate::control::pending_ddl::PendingDdlTable::new(), metadata_apply_wedge: std::sync::Arc::default(), sequencer_halt: std::sync::Arc::default(), @@ -293,6 +298,7 @@ impl SharedState { cross_shard_dlq: None, cross_shard_metrics: None, cross_shard_dedup: std::sync::OnceLock::new(), + redo_chunks, kafka_manager: crate::event::kafka::KafkaManager::new(shutdown.raw_receiver()), definition_sync_fanout: std::sync::Arc::new( crate::control::server::sync::definition_fanout::DefinitionSyncFanout::new(), diff --git a/nodedb/src/control/state/init_prod/open.rs b/nodedb/src/control/state/init_prod/open.rs index 25284d425..9084aec7b 100644 --- a/nodedb/src/control/state/init_prod/open.rs +++ b/nodedb/src/control/state/init_prod/open.rs @@ -112,6 +112,14 @@ impl SharedState { session_handles, } = super::stores::session_controls(auth_config, &rate_limit_config); + let redo_chunks = Arc::new( + crate::control::wal_replication::transaction_redo::RedoChunkStore::new( + Arc::clone(&wal), + crate::control::wal_replication::transaction_redo::chunks::RedoChunkLimits::from_tuning( + &tuning.calvin, + ), + ), + ); let hlc_clock = wal.hlc_clock(); let state = Arc::new(Self { outcome_floor: dispatcher.outcome_floor(), @@ -182,6 +190,7 @@ impl SharedState { cross_shard_dlq: None, cross_shard_metrics: None, cross_shard_dedup: std::sync::OnceLock::new(), + redo_chunks, kafka_manager: crate::event::kafka::KafkaManager::new(shutdown.raw_receiver()), definition_sync_fanout: std::sync::Arc::new( crate::control::server::sync::definition_fanout::DefinitionSyncFanout::new(), @@ -200,11 +209,8 @@ impl SharedState { ) .0, group_watchers: Arc::new(nodedb_cluster::GroupAppliedWatchers::new()), - metadata_ddl_lock: tokio::sync::Mutex::new(()), - metadata_ddl_owner: std::sync::Mutex::new(None), - metadata_ddl_applied_token: std::sync::atomic::AtomicU64::new(0), + metadata_ddl: crate::control::state::MetadataDdlState::new(), metadata_apply_progress: std::sync::atomic::AtomicU64::new(0), - metadata_ddl_token_seq: std::sync::atomic::AtomicU64::new(1), pending_ddl: crate::control::pending_ddl::PendingDdlTable::new(), metadata_apply_wedge: std::sync::Arc::default(), sequencer_halt: std::sync::Arc::default(), diff --git a/nodedb/src/control/state/metadata_ddl.rs b/nodedb/src/control/state/metadata_ddl.rs new file mode 100644 index 000000000..12da8ad24 --- /dev/null +++ b/nodedb/src/control/state/metadata_ddl.rs @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! This node's state of the replicated descriptor preparation lease. + +use std::sync::Mutex; +use std::sync::atomic::AtomicU64; + +use crate::control::metadata_proposer::DdlPrepareOwner; + +/// This node's state of the replicated descriptor preparation lease. +pub struct MetadataDdlState { + /// Serializes this node's attempts to acquire the replicated descriptor + /// preparation lease, and a local DDL through its post-apply. An async + /// proposer holds it across the post-apply, so it is a tokio mutex. + pub lock: tokio::sync::Mutex<()>, + /// Replicated preparation owner: its token, its node, and the local + /// monotonic apply time. The token and node are persisted in + /// `SystemCatalog` and seeded at boot. + pub owner: Mutex>, + /// Most recent fenced DDL token applied while its owner remained current. + /// Starts at 0 on boot. + pub applied_token: AtomicU64, + /// Per-node uniqueness component for descriptor-preparation lease tokens. + /// Starts at 1. + pub token_seq: AtomicU64, +} + +impl MetadataDdlState { + pub fn new() -> Self { + Self { + lock: tokio::sync::Mutex::new(()), + owner: Mutex::new(None), + applied_token: AtomicU64::new(0), + token_seq: AtomicU64::new(1), + } + } +} + +impl Default for MetadataDdlState { + fn default() -> Self { + Self::new() + } +} diff --git a/nodedb/src/control/state/mod.rs b/nodedb/src/control/state/mod.rs index 595b78ddc..602132929 100644 --- a/nodedb/src/control/state/mod.rs +++ b/nodedb/src/control/state/mod.rs @@ -13,6 +13,7 @@ mod fields; mod init; mod init_prod; mod init_variants; +mod metadata_ddl; mod methods; mod methods_audit; mod methods_lease; @@ -38,5 +39,6 @@ pub use self::calvin_cuts::{CalvinCuts, cut_instant_hook}; pub use self::calvin_local::CalvinLocalState; pub use self::fields::SharedState; pub use self::init_prod::DataPlaneHandles; +pub use self::metadata_ddl::MetadataDdlState; pub use self::tenant_request::TenantRequestGuard; pub use self::tenant_write::{TenantWriteMark, TenantWriteMarks, TenantWriteOrigin}; diff --git a/nodedb/src/control/wal_replication/decode/entry.rs b/nodedb/src/control/wal_replication/decode/entry.rs index aafc820ad..4232a36bd 100644 --- a/nodedb/src/control/wal_replication/decode/entry.rs +++ b/nodedb/src/control/wal_replication/decode/entry.rs @@ -229,5 +229,12 @@ fn to_physical_plan( ReplicatedWrite::SurrogateBind { .. } => Err(crate::Error::Internal { detail: "SurrogateBind reached to_physical_plan (should have been intercepted)".into(), }), + // The apply loop holds a redo stream's chunks itself; they reach no core. + ReplicatedWrite::RedoChunk { .. } => Err(crate::Error::Internal { + detail: "RedoChunk reached to_physical_plan (should have been intercepted)".into(), + }), + ReplicatedWrite::RedoAbandon { .. } => Err(crate::Error::Internal { + detail: "RedoAbandon reached to_physical_plan (should have been intercepted)".into(), + }), } } diff --git a/nodedb/src/control/wal_replication/decode/mod.rs b/nodedb/src/control/wal_replication/decode/mod.rs index f658d42b0..7dfdce432 100644 --- a/nodedb/src/control/wal_replication/decode/mod.rs +++ b/nodedb/src/control/wal_replication/decode/mod.rs @@ -40,4 +40,4 @@ mod vector_direct; pub use entry::{decode_parsed_entry, decode_replicated_entry, from_replicated_entry}; pub use entry_graph::edge_write_plan; -pub use transaction_redo::transaction_redo_payload; +pub use transaction_redo::{ChunkedRedo, DecodedTransactionRedo, decode_transaction_redo}; diff --git a/nodedb/src/control/wal_replication/decode/transaction_redo.rs b/nodedb/src/control/wal_replication/decode/transaction_redo.rs index 323379836..4e489728e 100644 --- a/nodedb/src/control/wal_replication/decode/transaction_redo.rs +++ b/nodedb/src/control/wal_replication/decode/transaction_redo.rs @@ -1,25 +1,86 @@ // SPDX-License-Identifier: BUSL-1.1 //! Decode a committed `ReplicatedWrite::TransactionRedo` entry back into the -//! payload every replica applies. +//! payload every replica applies, or into the stream a chunked body names. //! //! The apply loop intercepts these entries before the generic decode: the //! apply stamps the redo with the entry's Raft coordinates, which the generic //! `from_replicated_entry` result has no place for. +use nodedb_physical::physical_plan::RedoOrigin; use nodedb_types::Surrogate; use super::super::transaction_redo::TransactionRedoPayload; -use super::super::types::ReplicatedWrite; +use super::super::types::{RedoBody, RedoContent, ReplicatedEventSource, ReplicatedWrite}; use crate::control::surrogate::CarriedIdentity; +use crate::wal::RedoStreamId; -/// The payload `write` carries, or an error when `write` is another variant. -pub fn transaction_redo_payload(write: &ReplicatedWrite) -> crate::Result { +/// A decoded `TransactionRedo` entry. +#[derive(Debug, Clone)] +pub enum DecodedTransactionRedo { + /// The entry carries the whole payload. + Inline(Box), + /// The entry names the stream whose chunks carry the content. + Chunked(ChunkedRedo), +} + +/// The final entry of a chunked redo: its stream and the fields the entry +/// itself carries. +#[derive(Debug, Clone)] +pub struct ChunkedRedo { + pub stream: RedoStreamId, + /// How many chunks the stream holds. + pub count: u32, + /// How many bytes the chunks hold in all. + pub len: u64, + collections: Vec, + event_source: ReplicatedEventSource, + origin: RedoOrigin, +} + +impl ChunkedRedo { + /// The payload of the stream's assembled `content`. + pub fn payload(&self, content: RedoContent) -> TransactionRedoPayload { + payload_of(content, &self.collections, self.event_source, self.origin) + } + + /// Every collection the transaction wrote. + pub fn collections(&self) -> &[String] { + &self.collections + } +} + +fn payload_of( + content: RedoContent, + collections: &[String], + event_source: ReplicatedEventSource, + origin: RedoOrigin, +) -> TransactionRedoPayload { + TransactionRedoPayload { + redo: content.redo, + collections: collections.to_vec(), + sum_targets: content.sum_targets, + identities: content + .identities + .into_iter() + .map(|identity| CarriedIdentity { + collection: identity.collection, + pk_bytes: identity.pk_bytes, + surrogate: Surrogate::new(identity.surrogate), + }) + .collect(), + event_source: event_source.into(), + origin, + // The wire entry carries no Calvin meta. + calvin: None, + } +} + +/// What `write` carries, or an error when `write` is another variant. +pub fn decode_transaction_redo(write: &ReplicatedWrite) -> crate::Result { let ReplicatedWrite::TransactionRedo { - redo, + body, collections, - sum_targets, - identities, event_source, origin, } = write @@ -29,20 +90,21 @@ pub fn transaction_redo_payload(write: &ReplicatedWrite) -> crate::Result DecodedTransactionRedo::Inline(Box::new(payload_of( + (**content).clone(), + collections, + *event_source, + *origin, + ))), + RedoBody::Chunked { stream, count, len } => DecodedTransactionRedo::Chunked(ChunkedRedo { + stream: *stream, + count: *count, + len: *len, + collections: collections.clone(), + event_source: *event_source, + origin: *origin, + }), }) } @@ -50,11 +112,13 @@ pub fn transaction_redo_payload(write: &ReplicatedWrite) -> crate::Result TransactionRedoPayload { TransactionRedoPayload { @@ -97,6 +161,16 @@ mod tests { }], event_source: EventSource::Trigger, origin: RedoOrigin::Restore, + calvin: None, + } + } + + fn inline(write: &ReplicatedWrite) -> TransactionRedoPayload { + match decode_transaction_redo(write).expect("payload decodes") { + DecodedTransactionRedo::Inline(payload) => *payload, + DecodedTransactionRedo::Chunked(chunked) => { + panic!("expected an inline body, got stream {:?}", chunked.stream) + } } } @@ -116,7 +190,7 @@ mod tests { assert_eq!(decoded_entry.vshard_id, 7); assert_eq!(decoded_entry.idempotency_key, entry.idempotency_key); - let decoded = transaction_redo_payload(&decoded_entry.write).expect("payload decodes"); + let decoded = inline(&decoded_entry.write); assert_eq!(decoded.redo, original.redo); assert_eq!(decoded.collections, original.collections); assert_eq!(decoded.sum_targets, original.sum_targets); @@ -132,6 +206,51 @@ mod tests { assert_eq!(named, vec!["accounts", "entries"]); } + #[test] + fn a_chunked_final_entry_round_trips_and_rebuilds_the_payload() { + let mut original = payload(); + original.redo.ops[0].payload = vec![9; 200_000]; + let RedoProposal::Chunked { + stream, + chunks, + last, + } = session_redo_proposal( + RedoEntryTarget { + tenant_id: TenantId::new(1), + database_id: DatabaseId::new(5), + vshard_id: VShardId::new(7), + }, + &original, + 64 * 1024, + ) + .expect("build") + else { + panic!("a 200 KB redo travels chunked"); + }; + let decoded_last = ReplicatedEntry::from_bytes(&last.to_bytes()).expect("entry decodes"); + let DecodedTransactionRedo::Chunked(chunked) = + decode_transaction_redo(&decoded_last.write).expect("decodes") + else { + panic!("the final entry names its stream"); + }; + assert_eq!(chunked.stream, stream); + assert_eq!(chunked.count as usize, chunks.len()); + assert_eq!(chunked.collections(), original.collections.as_slice()); + let joined: Vec = chunks + .iter() + .flat_map(|chunk| match &chunk.write { + ReplicatedWrite::RedoChunk { bytes, .. } => bytes.clone(), + other => panic!("expected a chunk, got {other:?}"), + }) + .collect(); + let rebuilt = chunked.payload(RedoContent::from_bytes(&joined).expect("content")); + assert_eq!(rebuilt.redo, original.redo); + assert_eq!(rebuilt.sum_targets, original.sum_targets); + assert_eq!(rebuilt.identities, original.identities); + assert_eq!(rebuilt.event_source, EventSource::Trigger); + assert_eq!(rebuilt.origin, RedoOrigin::Restore); + } + #[test] fn apply_plan_carries_the_committed_redo_bytes_unchanged() { let original = payload(); @@ -153,6 +272,6 @@ mod tests { collection: "kv".into(), restart_identity: false, }; - assert!(transaction_redo_payload(&write).is_err()); + assert!(decode_transaction_redo(&write).is_err()); } } diff --git a/nodedb/src/control/wal_replication/encode/mod.rs b/nodedb/src/control/wal_replication/encode/mod.rs index 5ac14a582..ee35b5a40 100644 --- a/nodedb/src/control/wal_replication/encode/mod.rs +++ b/nodedb/src/control/wal_replication/encode/mod.rs @@ -24,4 +24,7 @@ mod transaction_redo; mod vector; pub use entry::to_replicated_entry; -pub use transaction_redo::transaction_redo_entry; +pub use transaction_redo::{ + CHUNK_ENTRY_ENVELOPE_BYTES, RedoEntryTarget, RedoProposal, chunk_entries, redo_abandon_entry, + session_redo_proposal, transaction_redo_entry, +}; diff --git a/nodedb/src/control/wal_replication/encode/transaction_redo.rs b/nodedb/src/control/wal_replication/encode/transaction_redo.rs index 44573c789..6f509959d 100644 --- a/nodedb/src/control/wal_replication/encode/transaction_redo.rs +++ b/nodedb/src/control/wal_replication/encode/transaction_redo.rs @@ -1,41 +1,83 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Encode a committed transaction's redo as a `ReplicatedWrite::TransactionRedo` -//! entry for its vShard's data-group log. +//! Encode a committed transaction's redo as `ReplicatedWrite::TransactionRedo` +//! entries for its vShard's data-group log. //! //! A redo is not built from a `PhysicalPlan`, so it has no arm in -//! `to_replicated_entry`: the commit that resolved it encodes it here. +//! `to_replicated_entry`: the commit that resolved it encodes it here. A redo +//! whose entry passes the redo entry limit travels as `RedoChunk` entries of +//! one stream, then a final entry whose body names the stream. use super::super::transaction_redo::TransactionRedoPayload; use super::super::types::{ - ReplicatedEntry, ReplicatedEventSource, ReplicatedIdentity, ReplicatedWrite, + RedoBody, RedoContent, ReplicatedEntry, ReplicatedEventSource, ReplicatedIdentity, + ReplicatedWrite, }; use crate::types::{DatabaseId, TenantId, VShardId}; +use crate::wal::RedoStreamId; -/// The Raft entry that carries `payload` to every replica of `vshard_id`. -pub fn transaction_redo_entry( - tenant_id: TenantId, - database_id: DatabaseId, - vshard_id: VShardId, +/// Room a chunk entry keeps for its envelope beside the chunk bytes: the +/// routing fields, the stream header, and the stamps the proposer adds. +pub const CHUNK_ENTRY_ENVELOPE_BYTES: usize = 4 * 1024; + +/// Room an inline entry keeps for its fixed fields and the stamps the +/// proposer adds after the size check: the commit HLC and the metadata floor. +const ENTRY_STAMP_BYTES: usize = 1024; + +/// Room each named collection's incarnation and wire framing take, beside +/// its name. +const COLLECTION_ENTRY_BYTES: usize = 64; + +/// Where a committed redo's entries go. +#[derive(Debug, Clone, Copy)] +pub struct RedoEntryTarget { + pub tenant_id: TenantId, + pub database_id: DatabaseId, + pub vshard_id: VShardId, +} + +/// How one committed redo travels through its vShard's data-group log. +pub enum RedoProposal { + /// One entry carries the whole redo. + Inline(ReplicatedEntry), + /// `chunks` carry the redo in order, then `last` names their stream. + Chunked { + stream: RedoStreamId, + chunks: Vec, + last: ReplicatedEntry, + }, +} + +/// The content of `payload`: the part a chunked redo splits. +fn content_of(payload: &TransactionRedoPayload) -> RedoContent { + RedoContent { + redo: payload.redo.clone(), + sum_targets: payload.sum_targets.clone(), + identities: payload + .identities + .iter() + .map(|identity| ReplicatedIdentity { + collection: identity.collection.clone(), + pk_bytes: identity.pk_bytes.clone(), + surrogate: identity.surrogate.as_u32(), + }) + .collect(), + } +} + +/// The `TransactionRedo` entry of `payload` with `body`. +fn redo_entry( + target: RedoEntryTarget, payload: &TransactionRedoPayload, + body: RedoBody, ) -> ReplicatedEntry { ReplicatedEntry::new( - tenant_id.as_u64(), - database_id.as_u64(), - vshard_id.as_u32(), + target.tenant_id.as_u64(), + target.database_id.as_u64(), + target.vshard_id.as_u32(), ReplicatedWrite::TransactionRedo { - redo: payload.redo.clone(), + body, collections: payload.collections.clone(), - sum_targets: payload.sum_targets.clone(), - identities: payload - .identities - .iter() - .map(|identity| ReplicatedIdentity { - collection: identity.collection.clone(), - pk_bytes: identity.pk_bytes.clone(), - surrogate: identity.surrogate.as_u32(), - }) - .collect(), event_source: ReplicatedEventSource::from(payload.event_source), origin: payload.origin, }, @@ -43,3 +85,215 @@ pub fn transaction_redo_entry( .with_event_source(payload.event_source) .naming(payload.collections.iter().map(String::as_str)) } + +/// The one Raft entry that carries `payload` to every replica of `vshard_id`. +pub fn transaction_redo_entry( + tenant_id: TenantId, + database_id: DatabaseId, + vshard_id: VShardId, + payload: &TransactionRedoPayload, +) -> ReplicatedEntry { + redo_entry( + RedoEntryTarget { + tenant_id, + database_id, + vshard_id, + }, + payload, + RedoBody::Inline(Box::new(content_of(payload))), + ) +} + +/// The entries of a session COMMIT's redo. The redo travels inline when its +/// entry fits `max_entry_bytes`, else chunked under a stream the final +/// entry's idempotency key names. +pub fn session_redo_proposal( + target: RedoEntryTarget, + payload: &TransactionRedoPayload, + max_entry_bytes: usize, +) -> crate::Result { + let content = content_of(payload); + let bytes = content.to_bytes()?; + // The entry around the content: the collections it names twice, once in + // the write and once with their incarnations, and the fixed fields. + let envelope = payload + .collections + .iter() + .map(|collection| 2 * collection.len() + COLLECTION_ENTRY_BYTES) + .sum::() + .saturating_add(ENTRY_STAMP_BYTES); + let mut last = redo_entry(target, payload, RedoBody::Inline(Box::new(content))); + if bytes.len().saturating_add(envelope) <= max_entry_bytes { + return Ok(RedoProposal::Inline(last)); + } + let stream = RedoStreamId::Session { + vshard: target.vshard_id.as_u32(), + idempotency_key: last.idempotency_key, + }; + let chunks = chunk_entries( + target, + stream, + &bytes, + max_entry_bytes, + payload.event_source, + )?; + let count = u32::try_from(chunks.len()).map_err(|_| crate::Error::Internal { + detail: format!( + "a redo of {} bytes splits into too many chunks", + bytes.len() + ), + })?; + last.write = ReplicatedWrite::TransactionRedo { + body: RedoBody::Chunked { + stream, + count, + len: bytes.len() as u64, + }, + collections: payload.collections.clone(), + event_source: ReplicatedEventSource::from(payload.event_source), + origin: payload.origin, + }; + Ok(RedoProposal::Chunked { + stream, + chunks, + last, + }) +} + +/// The `RedoChunk` entries that carry `bytes` under `stream`, each encoded +/// within `max_entry_bytes`. +pub fn chunk_entries( + target: RedoEntryTarget, + stream: RedoStreamId, + bytes: &[u8], + max_entry_bytes: usize, + event_source: crate::event::EventSource, +) -> crate::Result> { + let room = max_entry_bytes + .checked_sub(CHUNK_ENTRY_ENVELOPE_BYTES) + .filter(|room| *room > 0) + .ok_or_else(|| crate::Error::Config { + detail: format!( + "tuning.calvin.max_redo_entry_bytes {max_entry_bytes} leaves no room for a \ + redo chunk beside its {CHUNK_ENTRY_ENVELOPE_BYTES}-byte envelope" + ), + })?; + let len = bytes.len() as u64; + Ok((0u32..) + .zip(bytes.chunks(room)) + .map(|(index, part)| { + ReplicatedEntry::new( + target.tenant_id.as_u64(), + target.database_id.as_u64(), + target.vshard_id.as_u32(), + ReplicatedWrite::RedoChunk { + stream, + index, + len, + bytes: part.to_vec(), + }, + ) + .with_event_source(event_source) + }) + .collect()) +} + +/// The entry that drops `stream` on every replica. +pub fn redo_abandon_entry(target: RedoEntryTarget, stream: RedoStreamId) -> ReplicatedEntry { + ReplicatedEntry::new( + target.tenant_id.as_u64(), + target.database_id.as_u64(), + target.vshard_id.as_u32(), + ReplicatedWrite::RedoAbandon { stream }, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::event::EventSource; + use crate::wal::{RedoRecord, RedoSubRecord}; + use nodedb_physical::physical_plan::RedoOrigin; + + fn payload(op_bytes: usize) -> TransactionRedoPayload { + TransactionRedoPayload { + redo: RedoRecord { + version: 1, + ops: vec![RedoSubRecord { + record_type: nodedb_wal::record::RecordType::Put as u32, + payload: vec![7; op_bytes], + }], + calvin_stamp: None, + cross_shard_applied: None, + row_sources: Vec::new(), + publishes: Vec::new(), + row_changes: Vec::new(), + }, + collections: vec!["orders".into()], + sum_targets: Vec::new(), + identities: Vec::new(), + event_source: EventSource::User, + origin: RedoOrigin::Commit, + calvin: None, + } + } + + fn target() -> RedoEntryTarget { + RedoEntryTarget { + tenant_id: TenantId::new(1), + database_id: DatabaseId::DEFAULT, + vshard_id: VShardId::new(3), + } + } + + #[test] + fn a_redo_within_the_limit_travels_inline() { + let proposal = session_redo_proposal(target(), &payload(100), 64 * 1024).expect("build"); + assert!(matches!(proposal, RedoProposal::Inline(_))); + } + + #[test] + fn a_redo_past_the_limit_travels_in_chunks_that_fit_the_limit() { + let limit = 64 * 1024; + let proposal = session_redo_proposal(target(), &payload(300_000), limit).expect("build"); + let RedoProposal::Chunked { + stream, + chunks, + last, + } = proposal + else { + panic!("a 300 KB redo must travel chunked"); + }; + assert_eq!( + stream, + RedoStreamId::Session { + vshard: 3, + idempotency_key: last.idempotency_key + } + ); + assert!(chunks.len() >= 5); + for chunk in &chunks { + assert!(chunk.encode().expect("encode").len() + ENTRY_STAMP_BYTES <= limit); + } + let ReplicatedWrite::TransactionRedo { + body: RedoBody::Chunked { count, len, .. }, + .. + } = last.write + else { + panic!("the final entry names its stream"); + }; + assert_eq!(count as usize, chunks.len()); + let joined: Vec = chunks + .iter() + .flat_map(|chunk| match &chunk.write { + ReplicatedWrite::RedoChunk { bytes, .. } => bytes.clone(), + other => panic!("expected a chunk, got {other:?}"), + }) + .collect(); + assert_eq!(joined.len() as u64, len); + assert_eq!( + RedoContent::from_bytes(&joined).expect("content").redo, + payload(300_000).redo + ); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/abandon.rs b/nodedb/src/control/wal_replication/transaction_redo/abandon.rs new file mode 100644 index 000000000..c0d0dc723 --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/abandon.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Retry the abandon of a session redo stream until the stream closes. +//! +//! A session proposes the abandon of its stream once. When that proposal +//! fails, the stream joins the store's abandon queue. One task per node +//! retries each queued abandon while the stream is open on this node. +//! +//! The stream closes when its abandon or final entry applies, when its group +//! applies an entry of a later term, or when this node leaves the group. So +//! the proposer's term bounds the retries, not a clock. A stream that opens +//! here only after its abandon left the queue closes at its group's next +//! term. +//! +//! The task stops only at shutdown. Each queued stream still open then gets +//! a capture. + +use std::collections::BTreeMap; +use std::sync::{Arc, Weak}; +use std::time::Duration; + +use crate::control::shutdown::{ShutdownPhase, spawn_loop}; +use crate::control::state::SharedState; +use crate::control::wal_replication::encode::{RedoEntryTarget, redo_abandon_entry}; +use crate::control::wal_replication::propose_replicated_entry; +use crate::wal::RedoStreamId; + +/// How long one abandon proposal waits for its apply on this node. +const ATTEMPT_BUDGET: Duration = Duration::from_secs(5); + +/// Wait before the first retry. Each later wait doubles it, up to +/// [`MAX_BACKOFF`]. +const FIRST_BACKOFF: Duration = Duration::from_millis(50); + +const MAX_BACKOFF: Duration = Duration::from_secs(2); + +/// Propose the abandon of `stream` to `target`'s data group, and wait until +/// it applied on this node. +pub(super) async fn propose_abandon( + state: &SharedState, + target: RedoEntryTarget, + stream: RedoStreamId, +) -> crate::Result<()> { + let proposer = state.async_raft_proposer()?; + let entry = redo_abandon_entry(target, stream); + let deadline = tokio::time::Instant::now() + ATTEMPT_BUDGET; + propose_replicated_entry(state, proposer, entry, deadline) + .await + .map(|_| ()) +} + +/// Spawn the task that retries `shared`'s queued abandons. `start_raft` calls +/// it once the async raft proposer is installed. A second call spawns +/// nothing. +pub(crate) fn spawn_redo_abandoner(shared: &Arc) { + if !shared.redo_chunks.claim_abandoner() { + tracing::warn!("redo abandoner already running; start_raft appears to have run twice"); + return; + } + let weak = Arc::downgrade(shared); + let wake = shared.redo_chunks.abandon_wake(); + spawn_loop( + &shared.loop_registry, + &shared.shutdown, + "redo_abandoner", + ShutdownPhase::DrainingControlPlane, + move |mut shutdown| async move { + // The last error of each queued abandon that has not applied. + let mut failed: BTreeMap = BTreeMap::new(); + let mut backoff = FIRST_BACKOFF; + loop { + let Some(state) = weak.upgrade() else { + return; + }; + let due = state.redo_chunks.due_abandons(); + failed.retain(|stream, _| due.iter().any(|abandon| abandon.stream == *stream)); + for abandon in &due { + tokio::select! { + biased; + _ = shutdown.wait_cancelled() => { + give_up(&weak, &failed); + return; + } + result = propose_abandon(&state, abandon.target, abandon.stream) => { + match result { + Ok(()) => { + state.redo_chunks.settle_abandon(&abandon.stream); + failed.remove(&abandon.stream); + } + Err(error) => { + tracing::debug!( + stream = ?abandon.stream, + %error, + "redo stream abandon did not apply; retrying" + ); + failed.insert(abandon.stream, error); + } + } + } + } + } + drop(state); + if failed.is_empty() { + backoff = FIRST_BACKOFF; + tokio::select! { + biased; + _ = shutdown.wait_cancelled() => { + give_up(&weak, &failed); + return; + } + _ = wake.notified() => {} + } + } else { + tokio::select! { + biased; + _ = shutdown.wait_cancelled() => { + give_up(&weak, &failed); + return; + } + _ = tokio::time::sleep(backoff) => {} + _ = wake.notified() => {} + } + backoff = backoff.saturating_mul(2).min(MAX_BACKOFF); + } + } + }, + ); +} + +/// Record every queued abandon whose stream is still open: no retry runs +/// after this, so the stream closes only at its group's next term. +fn give_up(state: &Weak, failed: &BTreeMap) { + let Some(state) = state.upgrade() else { + return; + }; + for abandon in state.redo_chunks.due_abandons() { + tracing::warn!( + stream = ?abandon.stream, + group_id = abandon.group_id, + term = abandon.term, + "redo stream abandon given up at shutdown; the stream closes at its group's next term" + ); + crate::diag::redo_abandon_given_up(&abandon, failed.get(&abandon.stream)); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/abandon.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/abandon.rs new file mode 100644 index 000000000..442341ace --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/abandon.rs @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The session streams whose abandon proposal failed. +//! +//! The session proposes an abandon once. When that proposal fails, the +//! stream joins this queue, and the abandoner retries it while the stream is +//! open on this node. The stream closes once its proposer's term ends, so +//! the term bounds the retries. + +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use tokio::sync::Notify; + +use crate::control::wal_replication::encode::RedoEntryTarget; +use crate::wal::RedoStreamId; + +use super::store::RedoChunkStore; + +/// One queued abandon whose stream is open on this node. +#[derive(Debug, Clone, Copy)] +pub struct DueAbandon { + pub stream: RedoStreamId, + /// Where the abandon entry goes. + pub target: RedoEntryTarget, + /// The data group whose log carries the stream. + pub group_id: u64, + /// The Raft term of the stream's first chunk entry. + pub term: u64, + /// The stream's declared byte length. + pub len: u64, +} + +impl RedoChunkStore { + /// Queue the abandon of `stream` for the abandoner, and wake it. + pub fn queue_abandon(&self, stream: RedoStreamId, target: RedoEntryTarget) { + self.lock().abandons.insert(stream, target); + self.abandon_wake.notify_one(); + } + + /// The queued abandons whose stream is open. Drops every queued abandon + /// whose stream closed. + pub fn due_abandons(&self) -> Vec { + let mut state = self.lock(); + let state = &mut *state; + let streams = &state.streams; + state + .abandons + .retain(|stream, _| streams.contains_key(stream)); + state + .abandons + .iter() + .filter_map(|(stream, target)| { + streams.get(stream).map(|open| DueAbandon { + stream: *stream, + target: *target, + group_id: open.group_id, + term: open.term, + len: open.len, + }) + }) + .collect() + } + + /// Remove `stream` from the queue: its abandon applied on this node. + pub fn settle_abandon(&self, stream: &RedoStreamId) { + self.lock().abandons.remove(stream); + } + + /// Wakes the abandoner when an abandon is queued. + pub fn abandon_wake(&self) -> Arc { + Arc::clone(&self.abandon_wake) + } + + /// Claim the one abandoner task of this store. `false` when one runs. + pub fn claim_abandoner(&self) -> bool { + !self.abandoner.swap(true, Ordering::AcqRel) + } +} + +#[cfg(test)] +mod tests { + use super::super::store::tests::{chunk, session, wal}; + use super::super::store::{RedoChunkLimits, RedoChunkStore}; + use super::*; + use crate::types::{DatabaseId, TenantId, VShardId}; + + fn target() -> RedoEntryTarget { + RedoEntryTarget { + tenant_id: TenantId::new(1), + database_id: DatabaseId::DEFAULT, + vshard_id: VShardId::new(2), + } + } + + #[tokio::test] + async fn a_queued_abandon_is_due_while_its_stream_is_open() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RedoChunkStore::new( + wal(&dir), + RedoChunkLimits { + max_entry_bytes: 64 * 1024, + max_open_bytes: 1 << 20, + }, + ); + store + .apply_chunk(chunk(session(1), 0, 4, b"ab")) + .await + .expect("chunk"); + store.queue_abandon(session(1), target()); + // A stream that never opened here is not due. + store.queue_abandon(session(2), target()); + let due = store.due_abandons(); + assert_eq!(due.len(), 1); + assert_eq!(due[0].stream, session(1)); + assert_eq!((due[0].group_id, due[0].term, due[0].len), (1, 3, 4)); + // A later term of the group closes the stream, and its abandon with it. + store.end_terms_before(1, 4); + assert!(store.due_abandons().is_empty()); + assert!(store.lock().abandons.is_empty()); + assert!(store.claim_abandoner()); + assert!(!store.claim_abandoner()); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/carry.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/carry.rs new file mode 100644 index 000000000..54e58daff --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/carry.rs @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Carry a group's open streams in its Raft snapshot. +//! +//! A follower that installs a snapshot cut between a stream's chunks and its +//! final entry never applies those chunks. The snapshot carries every open +//! stream of the group, so the final entry finds the same stream on every +//! replica. + +use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; +use crate::wal::manager::{NO_APPLY_KEY, RedoChunkPlacement}; +use crate::wal::{CarriedRedoStream, RedoChunkHeader}; + +use super::error::RedoChunkError; +use super::store::RedoChunkStore; +use super::stream::OpenStream; + +impl RedoChunkStore { + /// Every open stream of `group_id`, for the group's snapshot. The caller + /// holds the group's apply fence. + pub fn capture_group(&self, group_id: u64) -> Vec { + self.lock() + .streams + .values() + .filter(|open| open.group_id == group_id) + .map(|open| CarriedRedoStream { + stream: open.stream, + term: open.term, + len: open.len, + chunks: open.chunks.clone(), + }) + .collect() + } + + /// Replace the open and parked streams of `group_id` with `carried`, + /// from the group's installed snapshot. The carried chunks are durable + /// in this node's WAL before they count, so a restart rebuilds them. + /// Runs after the install marker: boot drops the group's earlier streams + /// there. + pub async fn install_group( + &self, + group_id: u64, + carried: Vec, + ) -> Result<(), RedoChunkError> { + let (replaced, released) = { + let mut state = self.lock(); + let streams = std::mem::take(&mut state.streams); + let (replaced, kept): (Vec<_>, Vec<_>) = streams + .into_iter() + .partition(|(_, open)| open.group_id == group_id); + state.streams = kept.into_iter().collect(); + // The snapshot covers every final entry a parked stream waits + // for, so no re-apply needs its records. + let parked = std::mem::take(&mut state.parked); + let (released, kept): (Vec<_>, Vec<_>) = parked + .into_iter() + .partition(|parked| parked.group_id == group_id); + state.parked = kept; + self.note_open(&state); + (replaced, released) + }; + // The floor holds release outside the store lock. + drop(replaced); + drop(released); + let mut installed = Vec::with_capacity(carried.len()); + let mut last = None; + for stream in carried { + let placement = RedoChunkPlacement { + tenant_id: TenantId::new(0), + vshard_id: VShardId::new(stream.stream.vshard()), + database_id: DatabaseId::DEFAULT, + }; + let mut first: Option = None; + for (index, bytes) in (0u32..).zip(&stream.chunks) { + let header = RedoChunkHeader { + stream: stream.stream, + group_id, + term: stream.term, + len: stream.len, + index, + }; + let lsns = self + .wal + .append_redo_chunk(NO_APPLY_KEY, placement, header, bytes) + .map_err(|e| RedoChunkError::storage(stream.stream, e))?; + first.get_or_insert(lsns.first); + last = Some((stream.stream, lsns.last)); + } + if let Some(first) = first { + installed.push((stream, first)); + } + } + if let Some((stream, lsn)) = last { + self.wal + .wait_durable(lsn) + .await + .map_err(|e| RedoChunkError::storage(stream, e))?; + } + let mut state = self.lock(); + for (carried, first) in installed { + let held = carried.chunks.iter().map(|chunk| chunk.len() as u64).sum(); + let hold = self.wal.floor_holds().hold(first); + state.streams.insert( + carried.stream, + OpenStream { + stream: carried.stream, + group_id, + term: carried.term, + len: carried.len, + chunks: carried.chunks, + held, + hold, + }, + ); + } + self.note_open(&state); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::super::store::tests::{chunk, session, wal}; + use super::super::store::{RedoChunkLimits, RedoChunkStore}; + + fn limits() -> RedoChunkLimits { + RedoChunkLimits { + max_entry_bytes: 64 * 1024, + max_open_bytes: 1 << 20, + } + } + + #[tokio::test] + async fn a_carried_stream_replaces_the_followers_streams_and_survives_a_rebuild() { + let leader_dir = tempfile::tempdir().expect("tempdir"); + let leader = RedoChunkStore::new(wal(&leader_dir), limits()); + leader + .apply_chunk(chunk(session(1), 0, 5, b"abc")) + .await + .expect("chunk"); + leader + .apply_chunk(chunk(session(1), 1, 5, b"de")) + .await + .expect("chunk"); + let carried = leader.capture_group(1); + assert_eq!(carried.len(), 1); + + let follower_dir = tempfile::tempdir().expect("tempdir"); + let follower_wal = wal(&follower_dir); + let follower = RedoChunkStore::new(std::sync::Arc::clone(&follower_wal), limits()); + follower + .apply_chunk(chunk(session(9), 0, 2, b"zz")) + .await + .expect("stale chunk"); + follower_wal + .appender(0) + .append_snapshot_installed(1) + .expect("install marker"); + follower.install_group(1, carried).await.expect("install"); + assert!(follower.take_for_final(&session(9)).is_none()); + + let rebuilt = RedoChunkStore::new(std::sync::Arc::clone(&follower_wal), limits()); + rebuilt + .rebuild(&follower_wal.replay().expect("replay")) + .expect("rebuild"); + assert!(rebuilt.take_for_final(&session(9)).is_none()); + let open = rebuilt.take_for_final(&session(1)).expect("carried stream"); + assert_eq!(open.assemble(2, 5).expect("assembles"), b"abcde".to_vec()); + let live = follower + .take_for_final(&session(1)) + .expect("installed stream"); + assert_eq!(live.assemble(2, 5).expect("assembles"), b"abcde".to_vec()); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/error.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/error.rs new file mode 100644 index 000000000..bbb633d01 --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/error.rs @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Why a chunked redo stream refused an entry. + +use crate::bridge::envelope::ErrorCode; +use crate::wal::RedoStreamId; + +/// A chunked redo stream refused an entry, or its storage failed. +/// +/// Every variant but [`Self::Storage`] is a refusal: the log alone decides +/// it, so every replica refuses the same entry, and the entry applied +/// nothing. A refusal reaches the proposer as a retryable refusal. +#[derive(Debug, thiserror::Error)] +pub enum RedoChunkError { + #[error( + "redo stream {stream:?}: this node holds {open} bytes of open redo streams, and \ + {len} more pass the cap of {cap} (tuning.calvin.max_open_redo_bytes); retry once \ + open streams finish" + )] + StoreFull { + stream: RedoStreamId, + open: u64, + len: u64, + cap: u64, + }, + #[error("redo stream {stream:?} is not open on this node")] + NotOpen { stream: RedoStreamId }, + #[error("redo stream {stream:?}: chunk {index} arrived, chunk {expected} was next")] + OutOfOrder { + stream: RedoStreamId, + index: u32, + expected: u32, + }, + #[error("redo stream {stream:?}: a chunk of term {term} reached a stream of term {open_term}")] + TermChanged { + stream: RedoStreamId, + term: u64, + open_term: u64, + }, + #[error("redo stream {stream:?}: a chunk declares {declared} bytes, the stream {len}")] + LengthChanged { + stream: RedoStreamId, + declared: u64, + len: u64, + }, + #[error("redo stream {stream:?}: chunks hold {held} bytes, past the declared {len}")] + Overflow { + stream: RedoStreamId, + held: u64, + len: u64, + }, + #[error( + "redo stream {stream:?}: the final entry names {count} chunks of {len} bytes, the \ + stream holds {held_count} chunks of {held} bytes" + )] + Mismatch { + stream: RedoStreamId, + count: u32, + len: u64, + held_count: usize, + held: u64, + }, + #[error("redo stream {stream:?}: {source}")] + Storage { + stream: RedoStreamId, + source: Box, + }, +} + +impl RedoChunkError { + /// Whether the log decides this outcome alike on every replica. + pub fn is_refusal(&self) -> bool { + !matches!(self, Self::Storage { .. }) + } + + pub(super) fn storage(stream: RedoStreamId, source: crate::Error) -> Self { + Self::Storage { + stream, + source: Box::new(source), + } + } +} + +impl From for crate::Error { + fn from(error: RedoChunkError) -> Self { + match error { + RedoChunkError::Storage { source, .. } => *source, + refusal => crate::Error::DataPlane(ErrorCode::RetryableRefusal { + reason: refusal.to_string(), + }), + } + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/leave.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/leave.rs new file mode 100644 index 000000000..7f88dd464 --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/leave.rs @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Drop the streams of the data groups this node stops hosting. +//! +//! A node that leaves a group applies no later entry of it. No final entry, +//! abandon, or later term then closes the group's streams, so they close +//! here. Their floor holds release with them. +//! +//! A remount of the group resumes its old log, which can still name a +//! dropped stream. So each group that loses a stream owes a snapshot +//! install first (see [`super::owed`]). The debt is durable before any +//! floor hold releases: a checkpoint can truncate the stream's records once +//! the hold is gone. +//! +//! No WAL record marks the drop. A boot can rebuild a dropped stream from +//! its records, and the next membership pass drops it again. + +use std::collections::BTreeSet; + +use crate::control::security::catalog::SystemCatalog; + +use super::store::RedoChunkStore; + +impl RedoChunkStore { + /// Drop every open and parked stream of a data group `hosted` rejects, + /// once each such group owes a snapshot install in `catalog`. Returns + /// how many open streams dropped. + /// + /// The caller holds the lock that orders group mounts, so a group that + /// mounts concurrently keeps its streams, and a remount sees the debt. + /// When the debt does not persist, every stream stays. + pub fn drop_unhosted_groups( + &self, + hosted: impl Fn(u64) -> bool, + catalog: &SystemCatalog, + ) -> crate::Result { + let leaving: BTreeSet = { + let state = self.lock(); + state + .streams + .values() + .map(|open| open.group_id) + .chain(state.parked.iter().map(|parked| parked.group_id)) + .filter(|&group_id| !hosted(group_id)) + .collect() + }; + if leaving.is_empty() { + return Ok(0); + } + self.record_owed(catalog, &leaving)?; + // A stream of a group outside `leaving` opened after the scan. It + // drops on the next pass, once its group owes a snapshot. + let drops = |group_id: u64| leaving.contains(&group_id) && !hosted(group_id); + let (dropped, released) = { + let mut state = self.lock(); + let streams = std::mem::take(&mut state.streams); + let (dropped, kept): (Vec<_>, Vec<_>) = streams + .into_iter() + .partition(|(_, open)| drops(open.group_id)); + state.streams = kept.into_iter().collect(); + let parked = std::mem::take(&mut state.parked); + let (released, kept): (Vec<_>, Vec<_>) = parked + .into_iter() + .partition(|parked| drops(parked.group_id)); + state.parked = kept; + self.note_open(&state); + (dropped, released) + }; + let count = dropped.len(); + // The floor holds release outside the store lock. + drop(dropped); + drop(released); + Ok(count) + } +} + +#[cfg(test)] +mod tests { + use super::super::store::tests::{chunk, session, wal}; + use super::super::store::{RedoChunkLimits, RedoChunkStore}; + use super::*; + + fn limits() -> RedoChunkLimits { + RedoChunkLimits { + max_entry_bytes: 64 * 1024, + max_open_bytes: 1 << 20, + } + } + + fn catalog(dir: &tempfile::TempDir) -> SystemCatalog { + SystemCatalog::open(&dir.path().join("system.redb")).expect("catalog") + } + + #[tokio::test] + async fn leaving_a_group_drops_its_streams_and_releases_their_floor_holds() { + let dir = tempfile::tempdir().expect("tempdir"); + let catalog = catalog(&dir); + let store = RedoChunkStore::new(wal(&dir), limits()); + // Group 1: one open stream and one parked stream. + store + .apply_chunk(chunk(session(1), 0, 4, b"ab")) + .await + .expect("chunk"); + store + .apply_chunk(chunk(session(2), 0, 2, b"cd")) + .await + .expect("chunk"); + let failed = store.take_for_final(&session(2)).expect("open stream"); + store.park(failed); + // Group 9 stays hosted. + let mut kept = chunk(session(3), 0, 4, b"ef"); + kept.group_id = 9; + store.apply_chunk(kept).await.expect("chunk"); + assert_eq!(store.open_streams(), 2); + + let dropped = store + .drop_unhosted_groups(|group_id| group_id == 9, &catalog) + .expect("drop"); + assert_eq!(dropped, 1); + assert_eq!(store.open_streams(), 1); + assert!(store.take_for_final(&session(1)).is_none()); + // Only the hosted group's stream holds the floor. + assert!(store.wal.floor_holds().lowest().is_some()); + drop( + store + .take_for_final(&session(3)) + .expect("the hosted stream stays"), + ); + assert_eq!(store.wal.floor_holds().lowest(), None); + } + + /// A remount of a group that lost a stream resumes a log that can name + /// the stream. The group owes a snapshot install, durably, and a hosted + /// group owes nothing. + #[tokio::test] + async fn dropping_a_groups_streams_records_its_snapshot_requirement() { + let dir = tempfile::tempdir().expect("tempdir"); + let catalog = catalog(&dir); + let store = RedoChunkStore::new(wal(&dir), limits()); + assert_eq!( + store + .drop_unhosted_groups(|_| false, &catalog) + .expect("drop"), + 0 + ); + assert!( + store.owed_groups().is_empty(), + "a group with no stream owes nothing" + ); + + store + .apply_chunk(chunk(session(1), 0, 4, b"ab")) + .await + .expect("chunk"); + let failed = store.take_for_final(&session(1)).expect("open stream"); + store.park(failed); + let mut kept = chunk(session(3), 0, 4, b"ef"); + kept.group_id = 9; + store.apply_chunk(kept).await.expect("chunk"); + + store + .drop_unhosted_groups(|group_id| group_id == 9, &catalog) + .expect("drop"); + assert!(store.owes_snapshot(1), "a parked stream's group owes one"); + assert!(!store.owes_snapshot(9)); + assert_eq!(catalog.load_redo_snapshot_owed().expect("load"), vec![1]); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/mod.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/mod.rs new file mode 100644 index 000000000..16bcd0dc3 --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/mod.rs @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Chunked transaction redo: the streams a redo past the entry limit travels +//! in, held on every replica until the stream's final entry. +//! +//! - [`store`]: the node's open streams, leader admission, and stream close. +//! - [`stream`]: one open stream and its assembly. +//! - [`rebuild`]: boot rebuild from the WAL. +//! - [`carry`]: the open streams a group snapshot carries. +//! - [`leave`]: the drop of a left group's streams. +//! - [`owed`]: the groups whose replica here owes a snapshot install. +//! - [`abandon`]: the session streams whose abandon proposal failed. +//! - [`error`]: why a stream refused an entry. + +pub mod abandon; +pub mod carry; +pub mod error; +pub mod leave; +pub mod owed; +pub mod rebuild; +pub mod store; +pub mod stream; + +pub use abandon::DueAbandon; +pub use error::RedoChunkError; +pub use rebuild::RebuildStats; +pub use store::{ChunkApply, RedoChunkLimits, RedoChunkStore}; +pub use stream::OpenStream; diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/owed.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/owed.rs new file mode 100644 index 000000000..a3ee4317c --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/owed.rs @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The data groups whose replica on this node owes a snapshot install. +//! +//! A node that leaves a group drops the group's streams (see +//! [`super::leave`]). A remount resumes the group's old log. That log can +//! hold a later chunk or the final entry of a dropped stream. Here it +//! refuses, while every other replica installs the redo. So the drop marks +//! the group owed, durably, before the streams release their WAL floor +//! holds: +//! +//! - The group's snapshot requirement holds on this node while it is owed. +//! Its replica refuses log entries, and its leader sends a snapshot. The +//! snapshot carries the group's open streams. +//! - A refusal of a chunk or final entry of an owed group holds the group's +//! durable floor instead of passing the entry. +//! - A snapshot install of the group settles the debt. +//! +//! The set is a leaf lock: no other lock is taken while it is held. + +use std::collections::BTreeSet; +use std::sync::MutexGuard; + +use crate::control::security::catalog::SystemCatalog; + +use super::store::RedoChunkStore; + +impl RedoChunkStore { + fn owed_set(&self) -> MutexGuard<'_, BTreeSet> { + self.owed.lock().unwrap_or_else(|p| p.into_inner()) + } + + /// Load the owed groups the catalog saved, at boot. + pub fn load_owed(&self, groups: Vec) { + self.owed_set().extend(groups); + } + + /// Whether this node's replica of `group_id` owes a snapshot install. + pub fn owes_snapshot(&self, group_id: u64) -> bool { + self.owed_set().contains(&group_id) + } + + /// Every group whose replica here owes a snapshot install. + pub fn owed_groups(&self) -> Vec { + self.owed_set().iter().copied().collect() + } + + /// Record that the replicas of `groups` here owe a snapshot install. + /// Durable before it takes effect. A group owed already is not written + /// again. + pub fn record_owed( + &self, + catalog: &SystemCatalog, + groups: &BTreeSet, + ) -> crate::Result<()> { + let fresh: Vec = { + let owed = self.owed_set(); + groups + .iter() + .copied() + .filter(|group_id| !owed.contains(group_id)) + .collect() + }; + catalog.save_redo_snapshot_owed(&fresh)?; + self.owed_set().extend(fresh); + Ok(()) + } + + /// Settle the debt of `group_id`: a snapshot of the group installed + /// here, with the group's open streams. + pub fn settle_owed(&self, catalog: &SystemCatalog, group_id: u64) -> crate::Result<()> { + if !self.owes_snapshot(group_id) { + return Ok(()); + } + catalog.remove_redo_snapshot_owed(group_id)?; + self.owed_set().remove(&group_id); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::super::store::tests::wal; + use super::super::store::{RedoChunkLimits, RedoChunkStore}; + use super::*; + + fn limits() -> RedoChunkLimits { + RedoChunkLimits { + max_entry_bytes: 64 * 1024, + max_open_bytes: 1 << 20, + } + } + + #[test] + fn an_owed_group_survives_a_restart_until_its_snapshot_installs() { + let dir = tempfile::tempdir().expect("tempdir"); + let catalog = SystemCatalog::open(&dir.path().join("system.redb")).expect("catalog"); + let store = RedoChunkStore::new(wal(&dir), limits()); + + store + .record_owed(&catalog, &BTreeSet::from([3, 8])) + .expect("record"); + assert!(store.owes_snapshot(3)); + assert!(!store.owes_snapshot(4)); + + let restarted_dir = tempfile::tempdir().expect("tempdir"); + let restarted = RedoChunkStore::new(wal(&restarted_dir), limits()); + restarted.load_owed(catalog.load_redo_snapshot_owed().expect("load")); + assert_eq!(restarted.owed_groups(), vec![3, 8]); + + restarted.settle_owed(&catalog, 3).expect("settle"); + restarted + .settle_owed(&catalog, 4) + .expect("settle a group never owed"); + assert_eq!(restarted.owed_groups(), vec![8]); + assert_eq!(catalog.load_redo_snapshot_owed().expect("load"), vec![8]); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/rebuild.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/rebuild.rs new file mode 100644 index 000000000..a80695305 --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/rebuild.rs @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Rebuild the open streams from the WAL at boot. +//! +//! A chunk entry's records are durable before its entry concludes, and its +//! key rides its last piece, so a re-delivered chunk entry is skipped as a +//! repeat. The store must hold that chunk again from its records. Boot reads +//! every `RedoChunk` record in WAL order and keeps a stream open unless: +//! +//! - a `Closed` record names it (an abandon); +//! - a later snapshot install of its group replaced the group's streams; +//! - a session stream's final entry key is on a record (the final applied); +//! - a stamped transaction redo names a Calvin stream's position. +//! +//! A stream keeps its chunks from index `0` up to the first chunk with a +//! missing piece. That chunk's entry never concluded, so it applies again. + +use std::collections::{BTreeMap, HashSet}; + +use nodedb_wal::WalRecord; +use nodedb_wal::record::RecordType; + +use crate::types::Lsn; +use crate::wal::{RedoChunkHeader, RedoChunkRecord, RedoRecord, RedoStreamId}; + +use super::store::RedoChunkStore; +use super::stream::OpenStream; + +/// What a boot rebuild found. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct RebuildStats { + /// Streams the store holds open again. + pub open: usize, + /// Streams the WAL shows closed. + pub closed: usize, +} + +/// The pieces of one chunk read so far. +struct ChunkPieces { + pieces: u32, + got: BTreeMap>, + first: u64, +} + +/// One stream read so far. +struct Building { + group_id: u64, + term: u64, + len: u64, + chunks: BTreeMap, +} + +impl Building { + fn new(header: &RedoChunkHeader) -> Self { + Self { + group_id: header.group_id, + term: header.term, + len: header.len, + chunks: BTreeMap::new(), + } + } + + /// The whole chunks from index `0`, the first record's LSN, and the + /// bytes they hold. `None` when chunk `0` is not whole. + fn whole_prefix(self) -> Option<(Vec>, u64, u64)> { + let mut chunks = Vec::new(); + let mut first = u64::MAX; + let mut held = 0u64; + for (index, chunk) in self.chunks { + let whole = index as usize == chunks.len() + && chunk.got.len() == chunk.pieces as usize + && chunk.got.keys().copied().eq(0..chunk.pieces); + if !whole { + break; + } + first = first.min(chunk.first); + let bytes: Vec = chunk.got.into_values().flatten().collect(); + held += bytes.len() as u64; + chunks.push(bytes); + } + (!chunks.is_empty()).then_some((chunks, first, held)) + } +} + +impl RedoChunkStore { + /// Rebuild the open streams from `records`, the whole WAL in LSN order. + /// Runs at boot, before the data-group apply loop starts. + pub fn rebuild(&self, records: &[WalRecord]) -> crate::Result { + let mut building: BTreeMap = BTreeMap::new(); + let mut keys: HashSet = HashSet::new(); + let mut redo_records: Vec<&WalRecord> = Vec::new(); + let mut closed = 0usize; + for record in records { + let key = record.apply_key(); + if key != 0 { + keys.insert(key); + } + match RecordType::from_raw(record.logical_record_type()) { + Some(RecordType::RedoChunk) => { + match RedoChunkRecord::from_bytes(&record.payload)? { + RedoChunkRecord::Piece(piece) => { + let stream = building + .entry(piece.header.stream) + .or_insert_with(|| Building::new(&piece.header)); + let chunk = + stream + .chunks + .entry(piece.header.index) + .or_insert(ChunkPieces { + pieces: piece.pieces, + got: BTreeMap::new(), + first: record.header.lsn, + }); + chunk.first = chunk.first.min(record.header.lsn); + chunk.got.insert(piece.piece, piece.bytes); + } + RedoChunkRecord::Closed { stream } => { + closed += usize::from(building.remove(&stream).is_some()); + } + } + } + Some(RecordType::SnapshotInstalled) => { + if let Ok(raw) = <[u8; 8]>::try_from(record.payload.as_slice()) { + let group_id = u64::from_le_bytes(raw); + let before = building.len(); + building.retain(|_, stream| stream.group_id != group_id); + closed += before - building.len(); + } + } + Some(RecordType::TransactionRedo) => redo_records.push(record), + _ => {} + } + } + let applied_positions = if building + .keys() + .any(|stream| matches!(stream, RedoStreamId::Calvin { .. })) + { + stamped_positions(&redo_records)? + } else { + HashSet::new() + }; + let mut state = self.lock(); + let mut open = 0usize; + for (stream, built) in building { + let finished = match stream { + RedoStreamId::Session { + idempotency_key, .. + } => keys.contains(&idempotency_key), + RedoStreamId::Calvin { + vshard, + epoch, + position, + .. + } => applied_positions.contains(&(vshard, epoch, position)), + }; + if finished { + closed += 1; + continue; + } + let (group_id, term, len) = (built.group_id, built.term, built.len); + let Some((chunks, first, held)) = built.whole_prefix() else { + continue; + }; + let hold = self.wal.floor_holds().hold(Lsn::new(first)); + state.streams.insert( + stream, + OpenStream { + stream, + group_id, + term, + len, + chunks, + held, + hold, + }, + ); + open += 1; + } + self.note_open(&state); + Ok(RebuildStats { open, closed }) + } +} + +/// The `(vshard, epoch, position)` every stamped transaction redo names. +fn stamped_positions(records: &[&WalRecord]) -> crate::Result> { + let mut positions = HashSet::new(); + for record in records { + if let Some(stamp) = RedoRecord::from_bytes(&record.payload)?.calvin_stamp { + positions.insert((stamp.vshard_id, stamp.epoch, stamp.position)); + } + } + Ok(positions) +} + +#[cfg(test)] +mod tests { + use super::super::store::tests::{chunk, session, wal}; + use super::super::store::{RedoChunkLimits, RedoChunkStore}; + use super::*; + + fn limits() -> RedoChunkLimits { + RedoChunkLimits { + max_entry_bytes: 64 * 1024, + max_open_bytes: 1 << 20, + } + } + + #[tokio::test] + async fn boot_rebuilds_an_open_stream() { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = wal(&dir); + let live = RedoChunkStore::new(std::sync::Arc::clone(&wal), limits()); + let open = session(41); + let finished = session(42); + let abandoned = session(43); + for (index, part) in [&b"abc"[..], b"de"].into_iter().enumerate() { + live.apply_chunk(chunk(open, index as u32, 5, part)) + .await + .expect("chunk"); + } + live.apply_chunk(chunk(finished, 0, 2, b"xy")) + .await + .expect("chunk"); + live.apply_chunk(chunk(abandoned, 0, 2, b"zz")) + .await + .expect("chunk"); + // The finished stream's final entry appended a record under its key. + wal.appender(42) + .append_proposal_applied( + crate::types::TenantId::new(1), + crate::types::VShardId::new(2), + crate::types::DatabaseId::DEFAULT, + ) + .expect("final record"); + live.abandon(abandoned, 77, chunk(abandoned, 0, 0, b"").placement) + .await + .expect("abandon"); + wal.sync().expect("sync"); + + let rebuilt = RedoChunkStore::new(std::sync::Arc::clone(&wal), limits()); + let stats = rebuilt + .rebuild(&wal.replay().expect("replay")) + .expect("rebuild"); + assert_eq!(stats, RebuildStats { open: 1, closed: 2 }); + let stream = rebuilt.take_for_final(&open).expect("open stream rebuilt"); + assert_eq!(stream.assemble(2, 5).expect("assembles"), b"abcde".to_vec()); + assert!(rebuilt.take_for_final(&finished).is_none()); + assert!(rebuilt.take_for_final(&abandoned).is_none()); + } + + #[tokio::test] + async fn a_snapshot_install_drops_the_groups_earlier_streams() { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = wal(&dir); + let live = RedoChunkStore::new(std::sync::Arc::clone(&wal), limits()); + live.apply_chunk(chunk(session(5), 0, 2, b"ab")) + .await + .expect("chunk"); + wal.appender(0) + .append_snapshot_installed(1) + .expect("install marker"); + live.apply_chunk(chunk(session(6), 0, 2, b"cd")) + .await + .expect("chunk"); + wal.sync().expect("sync"); + + let rebuilt = RedoChunkStore::new(std::sync::Arc::clone(&wal), limits()); + rebuilt + .rebuild(&wal.replay().expect("replay")) + .expect("rebuild"); + assert!(rebuilt.take_for_final(&session(5)).is_none()); + assert!(rebuilt.take_for_final(&session(6)).is_some()); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/store.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/store.rs new file mode 100644 index 000000000..8919cd4aa --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/store.rs @@ -0,0 +1,546 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The node's open chunked redo streams. +//! +//! Every replica decides each chunk and final entry from the log alone: the +//! chunk order, the stream term and the declared lengths. No replica refuses +//! an entry for its own memory. The group leader bounds memory instead: its +//! write gate admits a stream's first chunk only while this node's open and +//! admitted stream bytes, plus the new stream's length, stay within +//! `tuning.calvin.max_open_redo_bytes`. A node then holds at most that cap +//! per data group it hosts. +//! +//! A stream closes when its final entry takes it, when its proposer abandons +//! it, when its group applies an entry of a later term, when its Calvin +//! position applies through another copy, or when this node stops hosting +//! its group. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Instant; + +use nodedb_types::config::tuning::CalvinTuning; +use tokio::sync::Notify; + +use crate::control::wal_replication::encode::RedoEntryTarget; +use crate::types::Lsn; +use crate::wal::manager::{RedoChunkPlacement, WalFloorHold}; +use crate::wal::{RedoChunkHeader, RedoStreamId, WalManager}; + +use super::error::RedoChunkError; +use super::stream::OpenStream; + +/// The size bounds of chunked redo. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RedoChunkLimits { + /// Largest encoded redo entry a proposer puts in a data-group log. + pub max_entry_bytes: usize, + /// Most bytes of open streams the leader admits on its node. + pub max_open_bytes: u64, +} + +impl RedoChunkLimits { + pub fn from_tuning(tuning: &CalvinTuning) -> Self { + Self { + max_entry_bytes: tuning.max_redo_entry_bytes, + max_open_bytes: tuning.max_open_redo_bytes, + } + } +} + +/// A first chunk the leader admitted that has not applied yet. +#[derive(Debug, Clone, Copy)] +pub(super) struct Pending { + pub(super) len: u64, + /// The proposer's deadline. Past it the admission counts no more. + pub(super) until: Instant, +} + +/// The floor of a stream whose final entry failed on this node. +pub(super) struct ParkedStream { + pub(super) group_id: u64, + /// Keeps the floor until the stream drops. + pub(super) _hold: WalFloorHold, +} + +#[derive(Default)] +pub(super) struct StoreState { + pub(super) streams: BTreeMap, + pub(super) pending: HashMap, + /// Streams whose final entry failed on this node. The next boot rebuilds + /// them from the records their floors keep. + pub(super) parked: Vec, + /// Session streams whose abandon proposal failed, and the target the + /// abandon entry goes to. The abandoner retries each while it is open. + pub(super) abandons: BTreeMap, +} + +/// One committed chunk entry, as the apply loop hands it over. +#[derive(Debug, Clone)] +pub struct ChunkApply { + pub group_id: u64, + /// The Raft term of the chunk entry. + pub term: u64, + /// The chunk entry's idempotency key. + pub apply_key: u64, + pub placement: RedoChunkPlacement, + pub stream: RedoStreamId, + pub index: u32, + /// The byte length of the whole stream. + pub len: u64, + pub bytes: Vec, +} + +/// The node's open chunked redo streams. +pub struct RedoChunkStore { + pub(super) wal: Arc, + limits: RedoChunkLimits, + pub(super) state: Mutex, + /// How many streams are open: the per-entry term check skips the lock + /// when none is. + open_count: AtomicUsize, + /// Wakes the abandoner when a failed abandon is queued. + pub(super) abandon_wake: Arc, + /// Whether an abandoner task runs for this store. + pub(super) abandoner: AtomicBool, + /// The data groups whose replica here owes a snapshot install (see + /// [`super::owed`]). A leaf lock, apart from `state`. + pub(super) owed: Mutex>, +} + +impl RedoChunkStore { + pub fn new(wal: Arc, limits: RedoChunkLimits) -> Self { + Self { + wal, + limits, + state: Mutex::new(StoreState::default()), + open_count: AtomicUsize::new(0), + abandon_wake: Arc::new(Notify::new()), + abandoner: AtomicBool::new(false), + owed: Mutex::new(BTreeSet::new()), + } + } + + pub fn limits(&self) -> RedoChunkLimits { + self.limits + } + + pub(super) fn lock(&self) -> MutexGuard<'_, StoreState> { + self.state.lock().unwrap_or_else(|p| p.into_inner()) + } + + /// Record how many streams `state` holds open. + pub(super) fn note_open(&self, state: &StoreState) { + self.open_count + .store(state.streams.len(), Ordering::Release); + } + + /// How many streams are open. + pub fn open_streams(&self) -> usize { + self.open_count.load(Ordering::Acquire) + } + + /// The declared bytes of every open stream. + pub fn open_bytes(&self) -> u64 { + self.lock().streams.values().map(|open| open.len).sum() + } + + /// Admit the first chunk of `stream`, declaring `len` bytes, on this + /// group leader. The admission counts until the chunk applies here or + /// `until` passes. A re-proposal of an admitted or open stream passes. + pub fn admit_stream( + &self, + stream: RedoStreamId, + len: u64, + until: Instant, + ) -> Result<(), RedoChunkError> { + let now = Instant::now(); + let mut state = self.lock(); + state.pending.retain(|_, pending| pending.until > now); + if state.streams.contains_key(&stream) { + return Ok(()); + } + if let Some(pending) = state.pending.get_mut(&stream) { + pending.until = pending.until.max(until); + return Ok(()); + } + let open = state.streams.values().map(|open| open.len).sum::() + + state + .pending + .values() + .map(|pending| pending.len) + .sum::(); + let cap = self.limits.max_open_bytes; + if open.saturating_add(len) > cap { + return Err(RedoChunkError::StoreFull { + stream, + open, + len, + cap, + }); + } + state.pending.insert(stream, Pending { len, until }); + Ok(()) + } + + /// Apply one chunk: check it against its stream, append its records + /// keyed with the entry's key, wait until they are durable, then hold it. + pub async fn apply_chunk(&self, chunk: ChunkApply) -> Result<(), RedoChunkError> { + self.check_chunk(&chunk)?; + let header = RedoChunkHeader { + stream: chunk.stream, + group_id: chunk.group_id, + term: chunk.term, + len: chunk.len, + index: chunk.index, + }; + let lsns = self + .wal + .append_redo_chunk(chunk.apply_key, chunk.placement, header, &chunk.bytes) + .map_err(|e| RedoChunkError::storage(chunk.stream, e))?; + self.wal + .wait_durable(lsns.last) + .await + .map_err(|e| RedoChunkError::storage(chunk.stream, e))?; + self.hold_chunk(chunk, lsns.first); + Ok(()) + } + + /// Refuse a chunk the stream cannot take next. + fn check_chunk(&self, chunk: &ChunkApply) -> Result<(), RedoChunkError> { + let stream = chunk.stream; + let bytes = chunk.bytes.len() as u64; + let state = self.lock(); + let Some(open) = state.streams.get(&stream) else { + if chunk.index != 0 { + return Err(RedoChunkError::NotOpen { stream }); + } + if bytes > chunk.len { + return Err(RedoChunkError::Overflow { + stream, + held: bytes, + len: chunk.len, + }); + } + return Ok(()); + }; + let expected = u32::try_from(open.chunks.len()).unwrap_or(u32::MAX); + if chunk.index != expected { + return Err(RedoChunkError::OutOfOrder { + stream, + index: chunk.index, + expected, + }); + } + if open.group_id != chunk.group_id || open.term != chunk.term { + return Err(RedoChunkError::TermChanged { + stream, + term: chunk.term, + open_term: open.term, + }); + } + if open.len != chunk.len { + return Err(RedoChunkError::LengthChanged { + stream, + declared: chunk.len, + len: open.len, + }); + } + if open.held.saturating_add(bytes) > open.len { + return Err(RedoChunkError::Overflow { + stream, + held: open.held.saturating_add(bytes), + len: open.len, + }); + } + Ok(()) + } + + /// Hold a durable chunk whose first record is at `first`. + fn hold_chunk(&self, chunk: ChunkApply, first: Lsn) { + let mut state = self.lock(); + let held = chunk.bytes.len() as u64; + match state.streams.get_mut(&chunk.stream) { + Some(open) => { + open.held += held; + open.chunks.push(chunk.bytes); + } + None => { + state.pending.remove(&chunk.stream); + let hold = self.wal.floor_holds().hold(first); + state.streams.insert( + chunk.stream, + OpenStream { + stream: chunk.stream, + group_id: chunk.group_id, + term: chunk.term, + len: chunk.len, + chunks: vec![chunk.bytes], + held, + hold, + }, + ); + } + } + self.note_open(&state); + } + + /// Take `stream` for its final entry. The final entry decides it from + /// the log, so it leaves the store whatever the final entry's outcome. + pub fn take_for_final(&self, stream: &RedoStreamId) -> Option { + let mut state = self.lock(); + let taken = state.streams.remove(stream); + self.note_open(&state); + taken + } + + /// Keep the records of a taken stream whose final entry failed on this + /// node, so the next boot rebuilds the stream for the entry's re-apply. + pub fn park(&self, open: OpenStream) { + let group_id = open.group_id; + let hold = open.into_hold(); + self.lock().parked.push(ParkedStream { + group_id, + _hold: hold, + }); + } + + /// Drop `stream` for its abandon entry, and make the drop durable under + /// the entry's key. + pub async fn abandon( + &self, + stream: RedoStreamId, + apply_key: u64, + placement: RedoChunkPlacement, + ) -> Result<(), RedoChunkError> { + { + let mut state = self.lock(); + state.streams.remove(&stream); + state.pending.remove(&stream); + self.note_open(&state); + } + let lsn = self + .wal + .append_redo_stream_closed(apply_key, placement, stream) + .map_err(|e| RedoChunkError::storage(stream, e))?; + self.wal + .wait_durable(lsn) + .await + .map_err(|e| RedoChunkError::storage(stream, e)) + } + + /// Drop every stream of `group_id` opened in a term before `term`: the + /// group applies an entry of `term`, so no earlier term adds an entry. + pub fn end_terms_before(&self, group_id: u64, term: u64) { + if self.open_streams() == 0 { + return; + } + let mut state = self.lock(); + state + .streams + .retain(|_, open| open.group_id != group_id || open.term >= term); + self.note_open(&state); + } + + /// Drop every stream of the Calvin position `(epoch, position)` of + /// `vshard`. The position applied through one copy of its redo, so no + /// other attempt's stream installs. The Calvin apply path calls it once + /// the position's install is durable. + pub fn drop_calvin_position(&self, vshard: u32, epoch: u64, position: u32) { + if self.open_streams() == 0 { + return; + } + let mut state = self.lock(); + state.streams.retain(|stream, _| { + !matches!( + stream, + RedoStreamId::Calvin { vshard: v, epoch: e, position: p, .. } + if *v == vshard && *e == epoch && *p == position + ) + }); + self.note_open(&state); + } +} + +#[cfg(test)] +pub(super) mod tests { + use super::*; + use crate::types::{DatabaseId, TenantId, VShardId}; + + pub(in super::super) fn wal(dir: &tempfile::TempDir) -> Arc { + Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("open wal")) + } + + pub(in super::super) fn session(key: u64) -> RedoStreamId { + RedoStreamId::Session { + vshard: 2, + idempotency_key: key, + } + } + + pub(in super::super) fn chunk( + stream: RedoStreamId, + index: u32, + len: u64, + bytes: &[u8], + ) -> ChunkApply { + ChunkApply { + group_id: 1, + term: 3, + apply_key: 100 + u64::from(index), + placement: RedoChunkPlacement { + tenant_id: TenantId::new(1), + vshard_id: VShardId::new(2), + database_id: DatabaseId::DEFAULT, + }, + stream, + index, + len, + bytes: bytes.to_vec(), + } + } + + fn limits(max_open_bytes: u64) -> RedoChunkLimits { + RedoChunkLimits { + max_entry_bytes: 64 * 1024, + max_open_bytes, + } + } + + #[tokio::test] + async fn chunks_reassemble_in_order() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RedoChunkStore::new(wal(&dir), limits(1 << 20)); + let stream = session(7); + for (index, part) in [&b"abc"[..], b"def", b"g"].into_iter().enumerate() { + store + .apply_chunk(chunk(stream, index as u32, 7, part)) + .await + .expect("chunk applies"); + } + assert_eq!(store.open_streams(), 1); + assert_eq!(store.open_bytes(), 7); + let open = store.take_for_final(&stream).expect("open stream"); + assert_eq!(open.assemble(3, 7).expect("assembles"), b"abcdefg".to_vec()); + assert_eq!(store.open_streams(), 0); + // The floor holds the stream's first record until the stream drops. + assert!(store.wal.floor_holds().lowest().is_some()); + drop(open); + assert_eq!(store.wal.floor_holds().lowest(), None); + } + + #[tokio::test] + async fn a_missing_chunk_refuses_the_final_entry() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RedoChunkStore::new(wal(&dir), limits(1 << 20)); + let stream = session(8); + store + .apply_chunk(chunk(stream, 0, 6, b"abc")) + .await + .expect("first chunk"); + let skipped = store.apply_chunk(chunk(stream, 2, 6, b"ghi")).await; + assert!(matches!( + skipped, + Err(RedoChunkError::OutOfOrder { + index: 2, + expected: 1, + .. + }) + )); + let open = store.take_for_final(&stream).expect("open stream"); + let refused = open.assemble(2, 6); + assert!(matches!(refused, Err(RedoChunkError::Mismatch { .. }))); + let error = crate::Error::from(refused.expect_err("refused")); + assert!(matches!( + error, + crate::Error::DataPlane(crate::bridge::envelope::ErrorCode::RetryableRefusal { .. }) + )); + // A later chunk of the taken stream finds no stream. + assert!(matches!( + store.apply_chunk(chunk(stream, 1, 6, b"def")).await, + Err(RedoChunkError::NotOpen { .. }) + )); + } + + #[tokio::test] + async fn a_new_stream_past_the_open_byte_cap_is_refused() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RedoChunkStore::new(wal(&dir), limits(10)); + let until = Instant::now() + std::time::Duration::from_secs(60); + store + .admit_stream(session(1), 6, until) + .expect("first stream fits"); + // A re-proposal of an admitted stream passes. + store + .admit_stream(session(1), 6, until) + .expect("re-proposal passes"); + let refused = store.admit_stream(session(2), 5, until); + assert!(matches!( + refused, + Err(RedoChunkError::StoreFull { + open: 6, + len: 5, + cap: 10, + .. + }) + )); + // The admitted stream applies, then finishes, and frees its bytes. + store + .apply_chunk(chunk(session(1), 0, 6, b"abcdef")) + .await + .expect("chunk"); + assert!(store.admit_stream(session(2), 5, until).is_err()); + drop(store.take_for_final(&session(1))); + store + .admit_stream(session(2), 5, until) + .expect("room after the first stream closed"); + // An admission past its deadline counts no more. + let other_dir = tempfile::tempdir().expect("tempdir"); + let store = RedoChunkStore::new(wal(&other_dir), limits(10)); + store + .admit_stream(session(3), 10, Instant::now()) + .expect("fits"); + store + .admit_stream(session(4), 10, until) + .expect("the expired admission frees its bytes"); + } + + #[tokio::test] + async fn a_later_term_drops_the_groups_earlier_streams() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RedoChunkStore::new(wal(&dir), limits(1 << 20)); + store + .apply_chunk(chunk(session(1), 0, 4, b"ab")) + .await + .expect("chunk"); + let mut other_group = chunk(session(2), 0, 4, b"ab"); + other_group.group_id = 9; + store.apply_chunk(other_group).await.expect("chunk"); + store.end_terms_before(1, 3); + assert_eq!(store.open_streams(), 2); + store.end_terms_before(1, 4); + assert_eq!(store.open_streams(), 1); + assert!(store.take_for_final(&session(2)).is_some()); + } + + #[tokio::test] + async fn an_applied_calvin_position_drops_every_attempt() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RedoChunkStore::new(wal(&dir), limits(1 << 20)); + for attempt in 0..2 { + let stream = RedoStreamId::Calvin { + vshard: 2, + epoch: 5, + position: 1, + attempt, + }; + store + .apply_chunk(chunk(stream, 0, 2, b"ab")) + .await + .expect("chunk"); + } + store.drop_calvin_position(2, 5, 1); + assert_eq!(store.open_streams(), 0); + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/stream.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/stream.rs new file mode 100644 index 000000000..29996111c --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/stream.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! One open chunked redo stream on this node. + +use crate::wal::RedoStreamId; +use crate::wal::manager::WalFloorHold; + +use super::error::RedoChunkError; + +/// The chunks one stream holds, and the WAL floor that keeps their records. +#[derive(Debug)] +pub struct OpenStream { + pub(super) stream: RedoStreamId, + /// The data group whose log carries the stream. + pub(super) group_id: u64, + /// The Raft term of the stream's first chunk entry. + pub(super) term: u64, + /// The byte length every chunk declares for the whole stream. + pub(super) len: u64, + /// The chunks, in index order. + pub(super) chunks: Vec>, + /// The bytes `chunks` hold. + pub(super) held: u64, + /// Keeps the stream's `RedoChunk` records until the stream closes. + pub(super) hold: WalFloorHold, +} + +impl OpenStream { + pub fn stream(&self) -> RedoStreamId { + self.stream + } + + /// The stream's bytes, when it holds exactly `count` chunks of `len` + /// bytes in all, as its final entry names. + pub fn assemble(&self, count: u32, len: u64) -> Result, RedoChunkError> { + if self.chunks.len() != count as usize || self.held != len || self.len != len { + return Err(RedoChunkError::Mismatch { + stream: self.stream, + count, + len, + held_count: self.chunks.len(), + held: self.held, + }); + } + let mut bytes = Vec::with_capacity(self.held as usize); + for chunk in &self.chunks { + bytes.extend_from_slice(chunk); + } + Ok(bytes) + } + + /// The floor guard, once the bytes are no longer needed. + pub(super) fn into_hold(self) -> WalFloorHold { + self.hold + } +} diff --git a/nodedb/src/control/wal_replication/transaction_redo/mod.rs b/nodedb/src/control/wal_replication/transaction_redo/mod.rs index d6c49df7f..883e9f801 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/mod.rs +++ b/nodedb/src/control/wal_replication/transaction_redo/mod.rs @@ -4,14 +4,23 @@ //! //! - [`payload`]: the redo a commit hands to the data-group log. //! - [`apply`]: the one path that appends and applies it on a node. +//! - [`chunks`]: the streams a redo past the entry limit travels in. +//! - [`propose`]: the proposal of a committed redo, inline or chunked. +//! - [`abandon`]: the retry of a failed stream abandon. //! - [`collections`] / [`sum_targets`]: what the payload derives from the //! commit's buffered plans. +pub mod abandon; pub mod apply; +pub mod chunks; pub mod collections; pub mod payload; +pub mod propose; pub mod sum_targets; +pub(crate) use abandon::spawn_redo_abandoner; pub use apply::RedoTarget; pub(crate) use apply::{enqueue_transaction_redo, record_cross_shard_key}; +pub use chunks::{RedoChunkError, RedoChunkStore}; pub use payload::TransactionRedoPayload; +pub(crate) use propose::propose_transaction_redo; diff --git a/nodedb/src/control/wal_replication/transaction_redo/payload.rs b/nodedb/src/control/wal_replication/transaction_redo/payload.rs index 9885faeb6..4c99e5342 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/payload.rs +++ b/nodedb/src/control/wal_replication/transaction_redo/payload.rs @@ -3,10 +3,13 @@ //! One committed transaction's redo for one vShard, as a commit hands it to //! the data-group log and as every replica applies it. -use nodedb_physical::physical_plan::{MetaOp, PhysicalPlan, RedoOrigin, RedoSumTargets}; +use nodedb_physical::physical_plan::{ + CalvinInstall, MetaOp, PhysicalPlan, RedoOrigin, RedoSumTargets, +}; use crate::control::state::SharedState; use crate::control::surrogate::{CarriedIdentity, collect_plan_identities}; +use crate::control::wal_replication::types::CalvinRedoMeta; use crate::event::EventSource; use crate::types::{DatabaseId, TenantId}; use crate::wal::RedoRecord; @@ -30,6 +33,9 @@ pub struct TransactionRedoPayload { pub event_source: EventSource, /// Which commit-boundary checks every replica's apply runs. pub origin: RedoOrigin, + /// Set when the redo is a committed Calvin slice. The record's + /// `calvin_stamp` names its `(epoch, position)`. + pub calvin: Option, } impl TransactionRedoPayload { @@ -55,6 +61,7 @@ impl TransactionRedoPayload { )?, event_source, origin: RedoOrigin::Commit, + calvin: None, }) } @@ -65,6 +72,109 @@ impl TransactionRedoPayload { collections: self.collections.clone(), sum_targets: self.sum_targets.clone(), origin: self.origin, + calvin: self.calvin_install()?, })) } + + /// The Calvin install the apply plan carries: the slice's stamp and the + /// meta beside it. A Calvin meta on a record with no stamp names no + /// position, so it is refused. + fn calvin_install(&self) -> crate::Result> { + let Some(meta) = &self.calvin else { + return Ok(None); + }; + let stamp = self + .redo + .calvin_stamp + .as_ref() + .ok_or_else(|| crate::Error::Internal { + detail: "a Calvin redo entry carries no calvin_stamp, so its install names no \ + (epoch, position); the entry is refused" + .into(), + })?; + Ok(Some(CalvinInstall { + epoch: stamp.epoch, + position: stamp.position, + epoch_system_ms: meta.epoch_system_ms, + reply: meta.reply.clone(), + })) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::wal::CalvinStamp; + use nodedb_physical::physical_plan::CalvinReplySpec; + + fn payload( + stamp: Option, + calvin: Option, + ) -> TransactionRedoPayload { + TransactionRedoPayload { + redo: RedoRecord { + version: 1, + ops: Vec::new(), + calvin_stamp: stamp, + cross_shard_applied: None, + row_sources: Vec::new(), + publishes: Vec::new(), + row_changes: Vec::new(), + }, + collections: vec!["orders".into()], + sum_targets: Vec::new(), + identities: Vec::new(), + event_source: EventSource::User, + origin: RedoOrigin::Commit, + calvin, + } + } + + fn meta() -> CalvinRedoMeta { + CalvinRedoMeta { + epoch_system_ms: 1_700, + reply: CalvinReplySpec::Count(vec![7]), + primary_write: true, + } + } + + fn install_of(payload: &TransactionRedoPayload) -> crate::Result> { + match payload.apply_plan()? { + PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { calvin, .. }) => Ok(calvin), + other => panic!("apply plan must be ApplyTransactionRedo, got {other:?}"), + } + } + + #[test] + fn a_session_redo_installs_without_a_calvin_install() { + let plain = payload(None, None); + assert_eq!(install_of(&plain).expect("plan"), None); + } + + #[test] + fn a_calvin_redo_installs_at_its_stamped_position() { + let stamp = CalvinStamp { + epoch: 9, + position: 4, + vshard_id: 2, + collections: Vec::new(), + sum_targets: Vec::new(), + }; + let calvin = payload(Some(stamp), Some(meta())); + assert_eq!( + install_of(&calvin).expect("plan"), + Some(CalvinInstall { + epoch: 9, + position: 4, + epoch_system_ms: 1_700, + reply: CalvinReplySpec::Count(vec![7]), + }) + ); + } + + #[test] + fn a_calvin_meta_without_a_stamp_is_refused() { + let unstamped = payload(None, Some(meta())); + assert!(install_of(&unstamped).is_err()); + } } diff --git a/nodedb/src/control/wal_replication/transaction_redo/propose.rs b/nodedb/src/control/wal_replication/transaction_redo/propose.rs new file mode 100644 index 000000000..988d067c0 --- /dev/null +++ b/nodedb/src/control/wal_replication/transaction_redo/propose.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Propose a committed session redo through its vShard's data-group log. +//! +//! A redo whose entry fits `tuning.calvin.max_redo_entry_bytes` travels as +//! one entry. A larger redo travels as chunk entries, proposed in order, each +//! applied on this node before the next, and then a final entry that names +//! their stream. A proposal that stops before its final entry applies +//! proposes an abandon, so every replica drops the stream's chunks. A failed +//! abandon proposal goes to the abandoner, which retries it while the stream +//! is open (see [`super::abandon`]). + +use crate::control::state::SharedState; +use crate::control::wal_replication::encode::{ + RedoEntryTarget, RedoProposal, session_redo_proposal, +}; +use crate::control::wal_replication::propose_replicated_entry; +use crate::types::Lsn; +use crate::wal::RedoStreamId; + +use super::abandon::propose_abandon; +use super::apply::RedoTarget; +use super::payload::TransactionRedoPayload; + +/// Propose `payload` to `target`'s data group and wait until it applied on +/// this node. Returns what [`propose_replicated_entry`] returns for the entry +/// that installs the redo. +pub(crate) async fn propose_transaction_redo( + state: &SharedState, + target: RedoTarget, + payload: &TransactionRedoPayload, + deadline: tokio::time::Instant, +) -> crate::Result<(Vec, Lsn)> { + let proposer = state.async_raft_proposer()?; + let entry_target = RedoEntryTarget { + tenant_id: target.tenant_id, + database_id: target.database_id, + vshard_id: target.vshard_id, + }; + let proposal = session_redo_proposal( + entry_target, + payload, + state.redo_chunks.limits().max_entry_bytes, + )?; + let (stream, chunks, last) = match proposal { + RedoProposal::Inline(entry) => { + return propose_replicated_entry(state, proposer, entry, deadline).await; + } + RedoProposal::Chunked { + stream, + chunks, + last, + } => (stream, chunks, last), + }; + for chunk in chunks { + if let Err(error) = propose_replicated_entry(state, proposer, chunk, deadline).await { + abandon(state, entry_target, stream).await; + return Err(error); + } + } + let installed = propose_replicated_entry(state, proposer, last, deadline).await; + if installed.is_err() { + abandon(state, entry_target, stream).await; + } + installed +} + +/// Propose the abandon of `stream` once. A stream the final entry already +/// took is gone, so the abandon then drops nothing. A failed proposal queues +/// the abandon for the abandoner, so the session's error returns now. +async fn abandon(state: &SharedState, target: RedoEntryTarget, stream: RedoStreamId) { + if let Err(error) = propose_abandon(state, target, stream).await { + tracing::warn!( + ?stream, + %error, + "redo stream abandon did not apply; the abandoner retries it while the stream is open" + ); + state.redo_chunks.queue_abandon(stream, target); + } +} diff --git a/nodedb/src/control/wal_replication/types/mod.rs b/nodedb/src/control/wal_replication/types/mod.rs index 7af91c755..d52e6a660 100644 --- a/nodedb/src/control/wal_replication/types/mod.rs +++ b/nodedb/src/control/wal_replication/types/mod.rs @@ -17,8 +17,10 @@ //! - [`replicated_write`]: the [`ReplicatedWrite`] enum itself. //! - [`replicated_entry`]: [`ReplicatedEntry`] (routing envelope) + (de)serialization. //! - [`transaction_redo_wire`]: wire types of the committed-transaction redo entry. +//! - [`redo_chunk_wire`]: a redo body, inline or chunked across entries. mod aliases; +mod redo_chunk_wire; mod replicated_entry; mod replicated_write; mod transaction_redo_wire; @@ -28,9 +30,10 @@ pub use aliases::{ AppliedWait, AsyncRaftProposer, AsyncRaftSubmit, ProposedAt, ProposedWrite, RaftAppliedIndexSink, RaftCompactor, RaftProposer, }; +pub use redo_chunk_wire::{RedoBody, RedoContent}; pub use replicated_entry::{CollectionIncarnation, ReplicatedEntry}; pub use replicated_write::ReplicatedWrite; -pub use transaction_redo_wire::{ReplicatedEventSource, ReplicatedIdentity}; +pub use transaction_redo_wire::{CalvinRedoMeta, ReplicatedEventSource, ReplicatedIdentity}; pub use wire_shapes::{ BalanceDeltaFields, ColumnarResolvedRow, ConstraintChangeOp, DocumentResolvedMutationWire, KvResolvedMutationWire, ReplicatedBatchEdge, ReplicatedSumTarget, diff --git a/nodedb/src/control/wal_replication/types/redo_chunk_wire.rs b/nodedb/src/control/wal_replication/types/redo_chunk_wire.rs new file mode 100644 index 000000000..3db3aa184 --- /dev/null +++ b/nodedb/src/control/wal_replication/types/redo_chunk_wire.rs @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Wire shapes of a committed redo body: inline, or chunked across entries. + +use nodedb_physical::physical_plan::RedoSumTargets; + +use super::transaction_redo_wire::ReplicatedIdentity; +use crate::wal::RedoStreamId; + +/// The row-sized part of a committed redo: what a chunked redo splits. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct RedoContent { + /// The resolved post-images. + pub redo: crate::wal::RedoRecord, + /// Materialized-sum resolution the document writes fold into their + /// targets, keyed by source collection. + pub sum_targets: Vec, + /// Identities every replica binds before the apply. + pub identities: Vec, +} + +impl RedoContent { + pub fn to_bytes(&self) -> crate::Result> { + zerompk::to_msgpack_vec(self).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("redo content encode: {e}"), + }) + } + + pub fn from_bytes(bytes: &[u8]) -> crate::Result { + zerompk::from_msgpack(bytes).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("redo content decode: {e}"), + }) + } +} + +/// Where a `TransactionRedo` entry's content is. +#[derive( + Debug, + Clone, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub enum RedoBody { + /// The entry carries the content. + Inline(Box), + /// Earlier `RedoChunk` entries of `stream` carry the content: `count` + /// chunks of `len` bytes in all, encoded as one [`RedoContent`]. + Chunked { + stream: RedoStreamId, + count: u32, + len: u64, + }, +} + +impl RedoBody { + /// Whether the body is a committed Calvin slice's. Its scheduler holds + /// its locks. + pub fn is_calvin(&self) -> bool { + match self { + Self::Inline(content) => content.redo.calvin_stamp.is_some(), + Self::Chunked { stream, .. } => matches!(stream, RedoStreamId::Calvin { .. }), + } + } +} diff --git a/nodedb/src/control/wal_replication/types/replicated_write.rs b/nodedb/src/control/wal_replication/types/replicated_write.rs index 814b58832..5c6067f54 100644 --- a/nodedb/src/control/wal_replication/types/replicated_write.rs +++ b/nodedb/src/control/wal_replication/types/replicated_write.rs @@ -5,11 +5,13 @@ use super::aliases::{ default_columnar_ingest_format, default_columnar_insert_intent, default_ivf_cells, default_ivf_nprobe, default_pq_m, }; +use super::redo_chunk_wire::RedoBody; use super::transaction_redo_wire::{ReplicatedEventSource, ReplicatedIdentity}; use super::wire_shapes::{ ColumnarResolvedRow, ConstraintChangeOp, DocumentResolvedMutationWire, KvResolvedMutationWire, ReplicatedBatchEdge, ReplicatedSumTarget, }; +use crate::wal::RedoStreamId; use nodedb_physical::physical_plan::document::MergeClauseOp; use nodedb_physical::physical_plan::{ ColumnarInsertIntent, CrdtWriteVerb, KvCounterShape, UpdateValue, VectorDirectWriteIntent, @@ -1029,21 +1031,17 @@ pub enum ReplicatedWrite { }, /// One committed transaction's resolved post-images for one vShard. /// - /// Every replica, the proposer included, appends `redo` to its own WAL - /// and applies it through the WAL replay arms, in Raft log order. The - /// record is resolved once, on the proposer; no replica re-derives it. - /// `redo.calvin_stamp` names the Calvin `(epoch, position)` the record - /// applies, when a Calvin flush produced it. + /// Every replica, the proposer included, appends the body's redo to its + /// own WAL and applies it through the WAL replay arms, in Raft log order. + /// The record is resolved once, on the proposer; no replica re-derives + /// it. The redo's `calvin_stamp` names the Calvin `(epoch, position)` the + /// record applies, when a Calvin slice produced it. A chunked body + /// assembles from the `RedoChunk` entries of its stream. TransactionRedo { - redo: crate::wal::RedoRecord, + body: RedoBody, /// Every collection the transaction wrote, for the collection-floor /// write versions. collections: Vec, - /// Materialized-sum resolution the document writes fold into their - /// targets, keyed by source collection. - sum_targets: Vec, - /// Identities every replica binds before the apply. - identities: Vec, event_source: ReplicatedEventSource, /// Which commit-boundary checks every replica's apply runs. origin: nodedb_physical::physical_plan::RedoOrigin, @@ -1086,6 +1084,20 @@ pub enum ReplicatedWrite { SurrogateBind { identities: Vec, }, + /// One chunk of a chunked redo's content. Every replica appends it to its + /// WAL and holds it until the stream's final `TransactionRedo` entry. + /// `len` is the byte length of the whole stream. + RedoChunk { + stream: RedoStreamId, + index: u32, + len: u64, + bytes: Vec, + }, + /// The proposer of `stream` gave up before its final entry. Every replica + /// drops the stream's held chunks. + RedoAbandon { + stream: RedoStreamId, + }, } #[cfg(test)] @@ -1235,6 +1247,23 @@ mod tests { constraint_version: 1, constraints: vec![vec![1, 2, 3]], }, + ReplicatedWrite::RedoChunk { + stream: RedoStreamId::Session { + vshard: 0, + idempotency_key: 9, + }, + index: 0, + len: 3, + bytes: vec![1, 2, 3], + }, + ReplicatedWrite::RedoAbandon { + stream: RedoStreamId::Calvin { + vshard: 0, + epoch: 4, + position: 1, + attempt: 0, + }, + }, ]; for write in writes { diff --git a/nodedb/src/control/wal_replication/types/transaction_redo_wire.rs b/nodedb/src/control/wal_replication/types/transaction_redo_wire.rs index ec9c736b8..1edd53a5d 100644 --- a/nodedb/src/control/wal_replication/types/transaction_redo_wire.rs +++ b/nodedb/src/control/wal_replication/types/transaction_redo_wire.rs @@ -2,8 +2,32 @@ //! Wire shapes carried by [`super::ReplicatedWrite::TransactionRedo`]. +use nodedb_physical::physical_plan::CalvinReplySpec; + use crate::event::EventSource; +/// What a committed Calvin slice's redo carries beyond the redo record. +/// The record's `calvin_stamp` names the slice's `(epoch, position)`. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CalvinRedoMeta { + /// The epoch's deterministic instant. + pub epoch_system_ms: i64, + /// The reply the leader's stage decided. Every replica renders it after + /// its install. + pub reply: CalvinReplySpec, + /// Whether the slice writes a row the statement names, not only a + /// derived row. + pub primary_write: bool, +} + /// One `(collection, primary key) → surrogate` identity a committed /// transaction's writes carry. Every replica binds it into its own catalog /// before the redo applies, so a later write by primary key on any replica diff --git a/nodedb/src/data/executor/dispatch/meta.rs b/nodedb/src/data/executor/dispatch/meta.rs index a0ed13128..d92529090 100644 --- a/nodedb/src/data/executor/dispatch/meta.rs +++ b/nodedb/src/data/executor/dispatch/meta.rs @@ -282,22 +282,28 @@ impl CoreLoop { } // Install a committed transaction's redo record through the WAL - // replay arms, on every replica, in Raft log order. + // replay arms, on every replica, in Raft log order. A committed + // Calvin slice also ends its staged state and answers its reply. MetaOp::ApplyTransactionRedo { redo, collections, sum_targets, origin, - } => self.install_redo( - task, - tid, - crate::data::executor::handlers::transaction::redo_apply::CommittedRedo { - redo, - collections, - sum_targets, - }, - *origin, - ), + calvin, + } => { + let committed = + crate::data::executor::handlers::transaction::redo_apply::CommittedRedo { + redo, + collections, + sum_targets, + }; + match calvin { + Some(install) => { + self.install_calvin_redo(task, tid, committed, *origin, install) + } + None => self.install_redo(task, tid, committed, *origin), + } + } // A RESTORE batch installs only inside its Calvin transaction: // its resolve appends the batch to the transaction's redo record. diff --git a/nodedb/src/data/executor/handlers/control/calvin/flush.rs b/nodedb/src/data/executor/handlers/control/calvin/flush.rs index 37e71f690..f6a618577 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/flush.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/flush.rs @@ -180,7 +180,9 @@ mod tests { assert_eq!(staged.status, Status::Ok, "{:?}", staged.error_code); let resolved = core.execute_calvin_resolve(&task, 1, 0); assert_eq!(resolved.status, Status::Ok, "{:?}", resolved.error_code); - resolved.payload.as_bytes().to_vec() + let answer: nodedb_physical::physical_plan::CalvinResolved = + zerompk::from_msgpack(resolved.payload.as_bytes()).expect("decode resolved answer"); + answer.redo } fn flush_at(core: &mut CoreLoop, lsn: u64, redo: &[u8], collections: &[String]) -> Response { diff --git a/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs b/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs index 89afdc475..9126c4b21 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs @@ -3,7 +3,7 @@ //! Test driver for a committed Calvin transaction on one core: stage the //! plans, resolve them into the redo record, and flush it. -use nodedb_physical::physical_plan::PhysicalPlan; +use nodedb_physical::physical_plan::{CalvinResolved, PhysicalPlan}; use crate::bridge::envelope::{Response, Status}; use crate::control::wal_replication::transaction_redo::collections::written_collections; @@ -58,7 +58,17 @@ impl CoreLoop { if resolved.status != Status::Ok { return resolved; } - let redo = resolved.payload.as_bytes().to_vec(); + let redo = match zerompk::from_msgpack::(resolved.payload.as_bytes()) { + Ok(answer) => answer.redo, + Err(error) => { + return self.response_error( + task, + crate::bridge::envelope::ErrorCode::Internal { + detail: format!("decode the resolved answer: {error}"), + }, + ); + } + }; let mut request = task.request.clone(); request.wal_lsn = Some(Lsn::new(lsn)); let flush_task = ExecutionTask::new(request); diff --git a/nodedb/src/data/executor/handlers/control/calvin_reply/mod.rs b/nodedb/src/data/executor/handlers/control/calvin_reply/mod.rs index f570f830b..6d35ff9f0 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_reply/mod.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_reply/mod.rs @@ -3,6 +3,7 @@ mod flush_read; mod images; mod reply; +mod spec; mod stage; mod target; diff --git a/nodedb/src/data/executor/handlers/control/calvin_reply/spec.rs b/nodedb/src/data/executor/handlers/control/calvin_reply/spec.rs new file mode 100644 index 000000000..7fcff0930 --- /dev/null +++ b/nodedb/src/data/executor/handlers/control/calvin_reply/spec.rs @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A staged reply as the redo entry carries it, and back. +//! +//! The leader's resolve turns the reply its stage decided into a +//! [`CalvinReplySpec`]. Every replica's install turns that spec back into a +//! [`CalvinReply`] and renders it. + +use nodedb_physical::physical_plan::{ + CalvinInstalledTimeseriesSpec, CalvinPostImagesSpec, CalvinReplyRow, CalvinReplySpec, + CalvinRowEngine, +}; +use nodedb_types::RowIdentity; + +use super::reply::{CalvinReply, InstalledTimeseries, PostImages}; +use super::target::RowEngine; + +impl From for CalvinRowEngine { + fn from(engine: RowEngine) -> Self { + match engine { + RowEngine::Document => Self::Document, + RowEngine::Crdt => Self::Crdt, + RowEngine::Kv => Self::Kv, + RowEngine::Vector => Self::Vector, + RowEngine::Columnar => Self::Columnar, + RowEngine::Timeseries => Self::Timeseries, + } + } +} + +impl From for RowEngine { + fn from(engine: CalvinRowEngine) -> Self { + match engine { + CalvinRowEngine::Document => Self::Document, + CalvinRowEngine::Crdt => Self::Crdt, + CalvinRowEngine::Kv => Self::Kv, + CalvinRowEngine::Vector => Self::Vector, + CalvinRowEngine::Columnar => Self::Columnar, + CalvinRowEngine::Timeseries => Self::Timeseries, + } + } +} + +impl From<&CalvinReply> for CalvinReplySpec { + fn from(reply: &CalvinReply) -> Self { + match reply { + CalvinReply::Count(payload) => Self::Count(payload.clone()), + CalvinReply::Rows(payload) => Self::Rows(payload.clone()), + CalvinReply::PostImages(images) => Self::PostImages(CalvinPostImagesSpec { + spec: images.spec.clone(), + rls_filters: images.rls_filters.clone(), + collection: images.collection.clone(), + engine: images.engine.into(), + rows: images + .rows + .iter() + .map(|(identity, surrogate)| CalvinReplyRow { + identity: identity.as_str().to_string(), + surrogate: *surrogate, + }) + .collect(), + }), + CalvinReply::InstalledTimeseries(installed) => { + Self::InstalledTimeseries(CalvinInstalledTimeseriesSpec { + spec: installed.spec.clone(), + rls_filters: installed.rls_filters.clone(), + collection: installed.collection.clone(), + ordinal: installed.ordinal as u64, + }) + } + } + } +} + +impl From for CalvinReply { + fn from(spec: CalvinReplySpec) -> Self { + match spec { + CalvinReplySpec::Count(payload) => Self::Count(payload), + CalvinReplySpec::Rows(payload) => Self::Rows(payload), + CalvinReplySpec::PostImages(images) => Self::PostImages(PostImages { + spec: images.spec, + rls_filters: images.rls_filters, + collection: images.collection, + engine: images.engine.into(), + rows: images + .rows + .into_iter() + .map(|row| (RowIdentity::from_user_key(row.identity), row.surrogate)) + .collect(), + }), + // An ordinal past the address space names no install, so the + // render reports the install absent. + CalvinReplySpec::InstalledTimeseries(installed) => { + Self::InstalledTimeseries(InstalledTimeseries { + spec: installed.spec, + rls_filters: installed.rls_filters, + collection: installed.collection, + ordinal: usize::try_from(installed.ordinal).unwrap_or(usize::MAX), + }) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_physical::physical_plan::{ReturningColumns, ReturningSpec}; + use nodedb_types::Surrogate; + + fn returning() -> ReturningSpec { + ReturningSpec { + columns: ReturningColumns::Star, + } + } + + fn round_trip(spec: CalvinReplySpec) { + let reply = CalvinReply::from(spec.clone()); + assert_eq!(CalvinReplySpec::from(&reply), spec); + } + + #[test] + fn every_reply_shape_round_trips_through_its_spec() { + round_trip(CalvinReplySpec::Count(vec![1, 2])); + round_trip(CalvinReplySpec::Rows(vec![3])); + round_trip(CalvinReplySpec::PostImages(CalvinPostImagesSpec { + spec: returning(), + rls_filters: vec![4], + collection: "orders".into(), + engine: CalvinRowEngine::Vector, + rows: vec![ + CalvinReplyRow { + identity: "o1".into(), + surrogate: Surrogate::new(1), + }, + CalvinReplyRow { + identity: "o2".into(), + surrogate: Surrogate::new(2), + }, + ], + })); + round_trip(CalvinReplySpec::InstalledTimeseries( + CalvinInstalledTimeseriesSpec { + spec: returning(), + rls_filters: Vec::new(), + collection: "metrics".into(), + ordinal: 3, + }, + )); + } +} diff --git a/nodedb/src/data/executor/handlers/control/calvin_resolve.rs b/nodedb/src/data/executor/handlers/control/calvin_resolve.rs index 8f36fac1f..452ee9ef7 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_resolve.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_resolve.rs @@ -13,8 +13,13 @@ //! [`CoreLoop::stage_calvin_overlay`][super::calvin_overlay_stage]). Reusing //! `execute_resolve_txn` directly means the redo serialization logic itself is //! never duplicated. +//! +//! The answer is a [`CalvinResolved`]: the redo record and the reply the +//! staged plans decided, as a [`nodedb_physical::physical_plan::CalvinReplySpec`]. + +use nodedb_physical::physical_plan::{CalvinReplySpec, CalvinResolved}; -use crate::bridge::envelope::Response; +use crate::bridge::envelope::{ErrorCode, Payload, Response, Status}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::task::ExecutionTask; @@ -22,8 +27,9 @@ use super::calvin_txn_id::calvin_synthetic_txn_id; impl CoreLoop { /// Resolve the Calvin transaction staged under `(epoch, position)` on - /// this vshard into a [`RedoRecord`][crate::wal::RedoRecord] and return - /// its encoded bytes, without touching any base engine. + /// this vshard into a [`RedoRecord`][crate::wal::RedoRecord], without + /// touching any base engine. Answers a [`CalvinResolved`] with the + /// record's encoded bytes and the staged reply. /// /// Errors (rather than silently dropping data or producing an empty /// record) when no `commit_pending` entry exists for @@ -48,11 +54,11 @@ impl CoreLoop { Err(e) => return self.response_error(task, e), }; - // Clone the staged plans (and the deterministic epoch anchor) out of - // `commit_pending` so the `&mut self` resolve call — which assigns - // bitemporal stamps into the overlay — does not overlap the immutable - // borrow of the pending buffer. - let (tid, plans, epoch_system_ms) = match self + // Clone the staged plans, the deterministic epoch anchor, and the + // staged reply out of `commit_pending` so the `&mut self` resolve + // call — which assigns bitemporal stamps into the overlay — does not + // overlap the immutable borrow of the pending buffer. + let (tid, plans, epoch_system_ms, reply) = match self .calvin .commit_pending .get(&(epoch, position, vshard_id)) @@ -61,6 +67,7 @@ impl CoreLoop { pending.tenant_id.as_u64(), pending.plans.clone(), pending.epoch_system_ms, + CalvinReplySpec::from(&pending.reply), ), None => { return self.response_error( @@ -89,10 +96,31 @@ impl CoreLoop { let prev_epoch_ms = self.epoch_system_ms; self.epoch_system_ms = Some(epoch_system_ms); self.apply_scope.calvin_txn_ordinal = Some(txn_ordinal); - let resp = self.execute_resolve_txn(task, tid, synthetic_txn_id, &plans); + let mut resp = self.execute_resolve_txn(task, tid, synthetic_txn_id, &plans); self.apply_scope.calvin_txn_ordinal = None; self.epoch_system_ms = prev_epoch_ms; - resp + if resp.status != Status::Ok { + return resp; + } + let resolved = CalvinResolved { + redo: resp.payload.as_bytes().to_vec(), + reply, + }; + match zerompk::to_msgpack_vec(&resolved) { + Ok(bytes) => { + resp.payload = Payload::from_vec(bytes); + resp + } + Err(error) => self.response_error( + task, + ErrorCode::Internal { + detail: format!( + "calvin resolve: encode the resolved answer of epoch={epoch} \ + position={position} vshard={vshard_id}: {error}" + ), + }, + ), + } } } @@ -219,6 +247,45 @@ mod tests { .expect("seed row"); } + /// The redo record a resolve answered with. + fn resolved_record(resp: &Response) -> RedoRecord { + let resolved: CalvinResolved = + zerompk::from_msgpack(resp.payload.as_bytes()).expect("decode resolved answer"); + RedoRecord::from_bytes(&resolved.redo).expect("decode redo record") + } + + /// The resolve answers the reply the stage decided, beside the record. + #[test] + fn a_resolve_answers_the_staged_reply() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let task = make_task(); + stage( + &mut core, + &task, + 2, + 0, + &[point_insert_plan("orders", "o1", 7)], + ); + let vshard_id = task.request.vshard_id.as_u32(); + let staged = CalvinReplySpec::from( + &core + .calvin + .commit_pending + .get(&(2, 0, vshard_id)) + .expect("staged commit") + .reply, + ); + + let resp = core.execute_calvin_resolve(&task, 2, 0); + + assert_eq!(resp.status, Status::Ok, "resolve must succeed: {resp:?}"); + let resolved: CalvinResolved = + zerompk::from_msgpack(resp.payload.as_bytes()).expect("decode resolved answer"); + assert_eq!(resolved.reply, staged); + assert_eq!(resolved_record(&resp).ops.len(), 1); + } + fn stage( core: &mut CoreLoop, task: &ExecutionTask, @@ -254,7 +321,7 @@ mod tests { let resp = core.execute_calvin_resolve(&task, 1, 0); assert_eq!(resp.status, Status::Ok, "resolve must succeed: {resp:?}"); - let record = RedoRecord::from_bytes(resp.payload.as_bytes()).expect("decode redo record"); + let record = resolved_record(&resp); assert!( record.calvin_stamp.is_none(), "calvin_stamp is filled in by a later unit, not resolve itself" @@ -320,7 +387,7 @@ mod tests { let resp = core.execute_calvin_resolve(&task, 3, 0); assert_eq!(resp.status, Status::Ok, "resolve must succeed: {resp:?}"); - let record = RedoRecord::from_bytes(resp.payload.as_bytes()).expect("decode redo record"); + let record = resolved_record(&resp); assert_eq!( record.ops.len(), 2, @@ -355,7 +422,7 @@ mod tests { let resp = core.execute_calvin_resolve(&task, 4, 0); assert_eq!(resp.status, Status::Ok, "resolve must succeed: {resp:?}"); - let record = RedoRecord::from_bytes(resp.payload.as_bytes()).expect("decode redo record"); + let record = resolved_record(&resp); assert_eq!( record.ops.len(), 2, @@ -387,7 +454,7 @@ mod tests { let resp = core.execute_calvin_resolve(&task, 5, 0); assert_eq!(resp.status, Status::Ok, "resolve must succeed: {resp:?}"); - let record = RedoRecord::from_bytes(resp.payload.as_bytes()).expect("decode redo record"); + let record = resolved_record(&resp); assert_eq!(record.ops.len(), 2, "one Put sub-record per predicted row"); // The expected post-image is the exact same decode -> apply -> encode @@ -507,7 +574,7 @@ mod tests { let resp = core.execute_calvin_resolve(&task, 6, 0); assert_eq!(resp.status, Status::Ok, "resolve must succeed: {resp:?}"); - let record = RedoRecord::from_bytes(resp.payload.as_bytes()).expect("decode redo"); + let record = resolved_record(&resp); let image = record .ops .iter() @@ -581,7 +648,7 @@ mod tests { core.apply_scope.calvin_txn_ordinal, None, "the resolve clears the transaction ordinal" ); - let record = RedoRecord::from_bytes(resp.payload.as_bytes()).expect("decode redo"); + let record = resolved_record(&resp); let put = record .ops .iter() diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_fold_tests.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_fold_tests.rs index abbb33c39..84bb5591f 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_fold_tests.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_fold_tests.rs @@ -19,18 +19,18 @@ use crate::types::Lsn; use crate::wal::{CalvinStamp, RedoRecord}; const DB: u64 = 0; -const TID: u64 = 1; +pub(super) const TID: u64 = 1; /// Source and target share a vShard, so the inline fold writes the target /// on this core. -const SOURCE: &str = "local_charges"; -const TARGET: &str = "local_balances"; +pub(super) const SOURCE: &str = "local_charges"; +pub(super) const TARGET: &str = "local_balances"; const ACCOUNT: &str = "a1"; -const TARGET_SURROGATE: Surrogate = Surrogate(4242); +pub(super) const TARGET_SURROGATE: Surrogate = Surrogate(4242); const LSN: u64 = 70; /// A core whose source collection folds `amount` into the target's /// `balance`, with the target row seeded at 100. -fn core_with_sum(dir: &std::path::Path) -> (CoreLoop, Box) { +pub(super) fn core_with_sum(dir: &std::path::Path) -> (CoreLoop, Box) { let (mut core, req, resp) = make_core_with_dir(dir); core.doc_configs.insert( (DatabaseId::DEFAULT, TenantId::new(TID), TARGET.to_string()), @@ -61,7 +61,7 @@ fn core_with_sum(dir: &std::path::Path) -> (CoreLoop, Box) { (core, Box::new((req, resp))) } -fn sum_targets() -> Vec { +pub(super) fn sum_targets() -> Vec { vec![RedoSumTargets { collection: SOURCE.to_string(), resolved: vec![ResolvedSumTarget::new(TARGET, ACCOUNT, TARGET_SURROGATE)], @@ -70,7 +70,7 @@ fn sum_targets() -> Vec { } /// A Calvin redo record writing one source row of `amount` 25. -fn calvin_redo() -> Vec { +pub(super) fn calvin_redo() -> Vec { let body = doc_format::encode_to_msgpack(&serde_json::json!({ "account_id": ACCOUNT, "amount": 25, @@ -94,7 +94,7 @@ fn calvin_redo() -> Vec { .expect("encode redo") } -fn balance(core: &CoreLoop) -> Option { +pub(super) fn balance(core: &CoreLoop) -> Option { let stored = core .sparse .get( diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install.rs new file mode 100644 index 000000000..fa37fa829 --- /dev/null +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install.rs @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The install of a committed Calvin slice from its stamped redo entry. +//! +//! Every replica runs it from the vShard's data-group log, in log order. The +//! install is the committed-redo install every transaction takes. Around it +//! the slice adds: +//! +//! 1. the slice's ordinal on the core clock, so a later local write stamps +//! above the record's graph versions; +//! 2. the end of the slice's staged state on a core that staged it: the +//! staged entry and its synthetic overlay go; +//! 3. the reply the leader's stage decided, rendered at the epoch instant +//! from the entry's [`CalvinReplySpec`], so every replica answers the same +//! bytes. +//! +//! The fold target rows stay in `Response::write_set`, so the funnel +//! journals them with the record. + +use nodedb_physical::physical_plan::{CalvinInstall, RedoOrigin}; +use tracing::info_span; + +use crate::bridge::envelope::{Payload, Response, Status}; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::control::calvin_reply::CalvinReply; +use crate::data::executor::task::ExecutionTask; + +use super::entry::CommittedRedo; + +impl CoreLoop { + /// Install the committed Calvin slice `install` names from its redo + /// record `committed`, and answer the slice's reply. + /// + /// A refused install rolled every write back and leaves any staged entry + /// in place. A reply that fails to render leaves the install `Ok`, since + /// the record installed. Its error travels in `error_code`. + pub(in crate::data::executor) fn install_calvin_redo( + &mut self, + task: &ExecutionTask, + tid: u64, + committed: CommittedRedo<'_>, + origin: RedoOrigin, + install: &CalvinInstall, + ) -> Response { + let CalvinInstall { + epoch, + position, + epoch_system_ms, + reply, + } = install; + let (epoch, position, epoch_system_ms) = (*epoch, *position, *epoch_system_ms); + let vshard_id = task.request.vshard_id.as_u32(); + let _apply_span = info_span!( + "executor_apply", + epoch, + position, + vshard = vshard_id, + tenant_id = tid, + trace_id = ?task.request.trace_id, + ) + .entered(); + // The record's graph versions stamp at least the slice's ordinal, so + // a later local write stamps above them. + let txn_ordinal = nodedb_types::calvin_txn_ordinal(epoch_system_ms, position); + self.hlc.update_from_remote(txn_ordinal); + + // What each resolved timeseries batch of the record stored: the + // reply answers the install's counts and stored rows. + let mut ts_installs = Vec::new(); + let mut response = + self.install_committed_redo_into(task, tid, committed, origin, &mut ts_installs); + if response.status != Status::Ok { + return response; + } + // The record installed: a staged entry of the slice and its overlay + // are spent. Writes waiting on the rows the entry owned run next. + let key = (epoch, position, vshard_id); + if self.calvin.commit_pending.remove(&key).is_some() { + self.calvin.fence.note_resolved(key, task.wal_lsn()); + } + self.drop_calvin_synthetic_overlay(epoch, position, vshard_id); + + let prev_epoch_ms = self.epoch_system_ms; + self.epoch_system_ms = Some(epoch_system_ms); + let rendered = + self.calvin_reply_payload(task, tid, CalvinReply::from(reply.clone()), &ts_installs); + self.epoch_system_ms = prev_epoch_ms; + match rendered { + Ok(payload) => response.payload = Payload::from_vec(payload), + Err(error) => response.error_code = Some(Box::new(error)), + } + response + } +} diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs new file mode 100644 index 000000000..fe5e55de2 --- /dev/null +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A committed Calvin slice installs from its stamped redo entry: the +//! `ApplyTransactionRedo` plan with a `CalvinInstall`. + +use nodedb_physical::physical_plan::{ + CalvinInstall, CalvinReplySpec, CalvinResolved, DocumentOp, MetaOp, PhysicalPlan, RedoOrigin, + RedoSumTargets, ReturningColumns, ReturningSpec, +}; +use nodedb_types::{QualifiedCollection, StorageKey, Surrogate}; + +use super::calvin_fold_tests::{TARGET, TID, balance, calvin_redo, core_with_sum, sum_targets}; +use crate::bridge::envelope::{Response, Status}; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; +use crate::data::executor::handlers::control::calvin::CalvinExecCtx; +use crate::data::executor::handlers::control::calvin_synthetic_txn_id; +use crate::data::executor::task::ExecutionTask; +use crate::types::{DatabaseId, Lsn, TenantId}; + +const ORDERS: &str = "orders"; +/// The vShard `make_default_task` homes to. +const VSHARD: u32 = 0; + +fn doc_body(field: &str, value: &str) -> Vec { + let mut obj = std::collections::HashMap::new(); + obj.insert( + field.to_string(), + nodedb_types::Value::String(value.to_string()), + ); + nodedb_types::value_to_msgpack(&nodedb_types::Value::Object(obj)).expect("encode body") +} + +/// An insert of row `surrogate` that asks for its row back. +fn returning_insert(document_id: &str, surrogate: u32) -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::PointInsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, ORDERS), + document_id: document_id.to_string(), + value: doc_body("a", "returned"), + if_absent: false, + surrogate: Surrogate::new(surrogate), + returning: Some(ReturningSpec { + columns: ReturningColumns::Star, + }), + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: Vec::new(), + }) +} + +/// Stage `plans` as the leader does at `(1, 0)` and resolve them. +fn stage_and_resolve(core: &mut CoreLoop, plans: &[PhysicalPlan]) -> CalvinResolved { + let task = make_default_task(); + let ctx = CalvinExecCtx { + epoch: 1, + position: 0, + epoch_system_ms: 0, + is_group_leader: true, + }; + let staged = + core.execute_calvin_execute_static(&task, ctx, &TenantId::new(TID), plans, &[], &[]); + assert_eq!(staged.status, Status::Ok, "{:?}", staged.error_code); + let resolved = core.execute_calvin_resolve(&task, 1, 0); + assert_eq!(resolved.status, Status::Ok, "{:?}", resolved.error_code); + zerompk::from_msgpack(resolved.payload.as_bytes()).expect("decode resolved answer") +} + +/// The stamped install of `redo` as the slice at `(epoch, 0)`. +fn install_plan( + redo: Vec, + collections: Vec, + sum_targets: Vec, + epoch: u64, + reply: CalvinReplySpec, +) -> PhysicalPlan { + PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { + redo, + collections, + sum_targets, + origin: RedoOrigin::Commit, + calvin: Some(CalvinInstall { + epoch, + position: 0, + epoch_system_ms: 0, + reply, + }), + }) +} + +/// Run `plan` on `core` with its record at `lsn`, through the plan +/// dispatcher. +fn run_at(core: &mut CoreLoop, plan: &PhysicalPlan, lsn: u64) -> Response { + let mut task: ExecutionTask = make_default_task(); + task.wal_lsn = Some(Lsn::new(lsn)); + core.execute_plan(&task, plan) +} + +fn base_row(core: &CoreLoop, surrogate: u32) -> Option> { + core.sparse + .get( + DatabaseId::DEFAULT.as_u64(), + TID, + ORDERS, + &StorageKey::for_surrogate(Surrogate::new(surrogate)), + ) + .expect("read base row") +} + +fn holds(payload: &[u8], needle: &[u8]) -> bool { + payload.windows(needle.len()).any(|w| w == needle) +} + +#[test] +fn a_calvin_install_consumes_its_staged_entry_and_renders_the_reply() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let resolved = stage_and_resolve(&mut core, &[returning_insert("o9", 9)]); + let synthetic = calvin_synthetic_txn_id(1, 0, VSHARD).expect("synthetic id"); + assert!(core.calvin.commit_pending.contains_key(&(1, 0, VSHARD))); + assert!(core.txn_overlays.contains_key(&synthetic)); + + let plan = install_plan( + resolved.redo, + vec![ORDERS.to_string()], + Vec::new(), + 1, + resolved.reply, + ); + let installed = run_at(&mut core, &plan, 42); + + assert_eq!(installed.status, Status::Ok, "{:?}", installed.error_code); + assert!(installed.error_code.is_none(), "{:?}", installed.error_code); + assert!( + !core.calvin.commit_pending.contains_key(&(1, 0, VSHARD)), + "the install consumes the staged entry" + ); + assert!( + !core.txn_overlays.contains_key(&synthetic), + "the install drops the synthetic overlay" + ); + assert!(base_row(&core, 9).is_some(), "the install writes the row"); + assert!( + holds(installed.payload.as_bytes(), b"returned"), + "the reply carries the RETURNING row" + ); +} + +#[test] +fn a_calvin_install_without_a_staged_entry_renders_the_same_reply() { + let leader_dir = tempfile::tempdir().expect("tempdir"); + let (mut leader, _ltx, _lrx) = make_core_with_dir(leader_dir.path()); + let resolved = stage_and_resolve(&mut leader, &[returning_insert("o9", 9)]); + let plan = install_plan( + resolved.redo, + vec![ORDERS.to_string()], + Vec::new(), + 1, + resolved.reply, + ); + let on_leader = run_at(&mut leader, &plan, 42); + assert_eq!(on_leader.status, Status::Ok, "{:?}", on_leader.error_code); + + // A replica that staged nothing installs the same entry. + let replica_dir = tempfile::tempdir().expect("tempdir"); + let (mut replica, _rtx, _rrx) = make_core_with_dir(replica_dir.path()); + assert!(replica.calvin.commit_pending.is_empty()); + let on_replica = run_at(&mut replica, &plan, 42); + + assert_eq!(on_replica.status, Status::Ok, "{:?}", on_replica.error_code); + assert!( + on_replica.error_code.is_none(), + "{:?}", + on_replica.error_code + ); + assert!( + base_row(&replica, 9).is_some(), + "the replica writes the row" + ); + assert!(holds(on_replica.payload.as_bytes(), b"returned")); + assert_eq!( + on_replica.payload.as_bytes(), + on_leader.payload.as_bytes(), + "every replica answers the same reply" + ); +} + +#[test] +fn a_calvin_install_keeps_fold_rows_in_the_write_set() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _ends) = core_with_sum(dir.path()); + let plan = install_plan( + calvin_redo(), + vec![super::calvin_fold_tests::SOURCE.to_string()], + sum_targets(), + 3, + CalvinReplySpec::Count(Vec::new()), + ); + + let installed = run_at(&mut core, &plan, 70); + + assert_eq!(installed.status, Status::Ok, "{:?}", installed.error_code); + assert_eq!(balance(&core).as_deref(), Some("125")); + assert!( + installed + .write_set + .iter() + .any(|entry| entry.collection.as_deref() == Some(TARGET)), + "the folded target row stays in the write set for the funnel to journal: {:?}", + installed.write_set + ); +} diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/mod.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/mod.rs index 63ffb21aa..0bb273351 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/mod.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/mod.rs @@ -4,6 +4,7 @@ //! the core that owns its vShard, through the WAL replay arms. //! //! - [`entry`]: the `MetaOp::ApplyTransactionRedo` handler. +//! - [`calvin_install`]: the install of a committed Calvin slice. //! - [`validate`]: commit-boundary checks that run before any write. //! - [`passes`]: the validate and install passes over the replay arms. //! - [`settle`]: the work an install defers until every sub-record landed. @@ -16,10 +17,15 @@ //! engine kind. //! - [`calvin_fold_tests`]: a Calvin record folds the same live and in //! restart replay. +//! - [`calvin_install_tests`]: a Calvin slice installs from its stamped +//! redo entry. //! - [`unique_handover_tests`]: UNIQUE judged on a record's post-state. #[cfg(test)] mod calvin_fold_tests; +mod calvin_install; +#[cfg(test)] +mod calvin_install_tests; mod document; mod entry; mod events; diff --git a/nodedb/src/diag/context/mod.rs b/nodedb/src/diag/context/mod.rs index 1a26fa9f0..01f169304 100644 --- a/nodedb/src/diag/context/mod.rs +++ b/nodedb/src/diag/context/mod.rs @@ -20,6 +20,7 @@ mod outcome_floor; mod quota; mod raft_apply; mod recovery; +mod redo_stream; mod retention; mod vector; mod vector_build; @@ -52,6 +53,9 @@ pub(in crate::diag) use quota::{ }; pub(in crate::diag) use raft_apply::{RaftEntryReapplied, ReplicatedWriteParked}; pub(in crate::diag) use recovery::{ReplayRecordUnapplied, WalArchivalFailedTruncationHeld}; +pub(in crate::diag) use redo_stream::{ + RedoAbandonGivenUp, RedoSnapshotDebtNotRecorded, RedoStreamLostHere, +}; pub(in crate::diag) use retention::RetentionAutowireOrphaned; pub(in crate::diag) use vector::VectorIndexNotApplied; pub(in crate::diag) use vector_build::{VectorBuildNotInstalled, VectorBuilderUnavailable}; diff --git a/nodedb/src/diag/context/redo_stream.rs b/nodedb/src/diag/context/redo_stream.rs new file mode 100644 index 000000000..c3ae03284 --- /dev/null +++ b/nodedb/src/diag/context/redo_stream.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Forensic payloads for chunked redo streams: an abandon that stopped +//! retrying, and a stream entry held on a replica that dropped the stream. + +use faultbox::DomainContext; +use faultbox::serde_json::{Value, json}; + +/// The abandoner stopped retrying a session stream's abandon while the +/// stream was open on this node. +pub(in crate::diag) struct RedoAbandonGivenUp { + /// The stream, as its `Debug` text. + pub stream: String, + pub vshard: u32, + /// The data group whose log carries the stream. + pub group_id: u64, + /// The Raft term of the stream's first chunk entry. + pub term: u64, + /// The stream's declared byte length. + pub len: u64, + /// The last abandon proposal's error, when one ran. + pub last_error: Option, +} + +impl DomainContext for RedoAbandonGivenUp { + fn domain_kind(&self) -> &'static str { + "nodedb.redo_abandon_given_up" + } + + fn grouping_key(&self) -> String { + // One bug class: an abandon that never applied before the node + // stopped. The stream and its group are the occurrence. + "redo_abandon_given_up".to_string() + } + + fn to_json(&self) -> Value { + json!({ + "stream": self.stream, + "vshard": self.vshard, + "group_id": self.group_id, + "term": self.term, + "len": self.len, + "last_error": self.last_error, + "impact": "every replica holds the stream's chunks and a WAL floor at its \ + first chunk record. They stay until the group applies an entry \ + of a later term, so the WAL on each replica cannot truncate past \ + that record meanwhile", + "operator_action": "a leadership transfer of the named group ends the term \ + and drops the stream on every replica. The last error \ + names why the abandon proposal did not apply", + }) + } +} + +/// A chunk or final entry of a redo stream refused on a replica that owes +/// its group a snapshot install: this node dropped the group's streams when +/// it left the group, and a remount resumed the group's old log. +pub(in crate::diag) struct RedoStreamLostHere { + /// The data group whose log carries the stream. + pub group_id: u64, + /// The held entry's index in the group's log. + pub log_index: u64, + /// The stream, as its `Debug` text. + pub stream: String, + pub vshard: u32, + /// `chunk` or `final`. + pub entry: &'static str, + /// Why this replica's stream refused the entry. + pub refusal: String, +} + +impl DomainContext for RedoStreamLostHere { + fn domain_kind(&self) -> &'static str { + "nodedb.redo_stream_lost_here" + } + + fn grouping_key(&self) -> String { + // One bug class: a log resumed past streams this replica dropped. + // The group, stream, and index are the occurrence. + "redo_stream_lost_here".to_string() + } + + fn to_json(&self) -> Value { + json!({ + "group_id": self.group_id, + "log_index": self.log_index, + "stream": self.stream, + "vshard": self.vshard, + "entry": self.entry, + "refusal": self.refusal, + "impact": "the other replicas can install the stream's redo, which this replica \ + lacks. Its durable floor holds at the entry, so a restart applies the \ + entry again. Later entries of the group apply here on the state \ + without the redo until a snapshot of the group installs", + "operator_action": "the node's membership pass makes the group's replica here \ + refuse log entries, and moves a leadership it holds. The \ + group's leader then sends a snapshot. When the group has no \ + other replica, no snapshot comes: restore the group from a \ + backup", + }) + } +} + +/// The membership pass failed to persist that data groups this node left +/// owe a snapshot install, so their redo streams stay. +pub(in crate::diag) struct RedoSnapshotDebtNotRecorded { + /// The catalog error. + pub error: String, +} + +impl DomainContext for RedoSnapshotDebtNotRecorded { + fn domain_kind(&self) -> &'static str { + "nodedb.redo_snapshot_debt_not_recorded" + } + + fn grouping_key(&self) -> String { + // The pass retries on every tick: one report per root cause. + "redo_snapshot_debt_not_recorded".to_string() + } + + fn to_json(&self) -> Value { + json!({ + "error": self.error, + "impact": "the left groups' redo streams and their WAL floor holds stay on this \ + node, so the WAL cannot truncate past their first chunk records", + "operator_action": "the error names why the system catalog refused the write. \ + The pass retries on every membership tick", + }) + } +} diff --git a/nodedb/src/diag/mod.rs b/nodedb/src/diag/mod.rs index c2f3ec656..6f77c4f82 100644 --- a/nodedb/src/diag/mod.rs +++ b/nodedb/src/diag/mod.rs @@ -20,6 +20,7 @@ pub use recording::{ index_rebuild_not_installed, metadata_apply_wedged, orphaned_index_entry_after_delete, quota_row_invalid, quota_row_undecodable, quota_row_write_failed, quota_scope_purge_incomplete, quota_scope_replay_aborted, raft_entries_reapplied, raft_entry_reapplied, + redo_abandon_given_up, redo_snapshot_debt_not_recorded, redo_stream_lost_here, replay_record_unapplied, replicated_write_parked, replicated_writes_parked, retention_autowire_orphaned, scope_quota_not_installed, strict_row_image_unrendered, strict_row_undecodable, synonym_group_not_applied, timeseries_partition_unreadable, diff --git a/nodedb/src/diag/recording/mod.rs b/nodedb/src/diag/recording/mod.rs index c34f9d202..e5130adae 100644 --- a/nodedb/src/diag/recording/mod.rs +++ b/nodedb/src/diag/recording/mod.rs @@ -21,6 +21,7 @@ mod outcome_floor; mod quota; mod raft_apply; mod recovery; +mod redo_stream; mod retention; mod shared; mod vector; @@ -57,6 +58,9 @@ pub use recovery::{ replay_record_unapplied, strict_row_undecodable, wal_archival_failed_truncation_held, write_acked_without_durability, }; +pub use redo_stream::{ + redo_abandon_given_up, redo_snapshot_debt_not_recorded, redo_stream_lost_here, +}; pub use retention::retention_autowire_orphaned; pub use shared::entry_kind; pub use vector::vector_index_not_applied; diff --git a/nodedb/src/diag/recording/redo_stream.rs b/nodedb/src/diag/recording/redo_stream.rs new file mode 100644 index 000000000..e0c73936e --- /dev/null +++ b/nodedb/src/diag/recording/redo_stream.rs @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Capture sites for chunked redo streams: an abandon that stopped retrying, +//! and a stream entry held on a replica that dropped the stream. + +use faultbox::{Capture, EventKind, error_chain_of}; + +use crate::control::wal_replication::transaction_redo::chunks::{DueAbandon, RedoChunkError}; +use crate::diag::context; +use crate::wal::RedoStreamId; + +/// Report a session stream left open when the abandoner stopped. Called from +/// the abandoner's exit, the one site that gives the abandon up. +pub fn redo_abandon_given_up(abandon: &DueAbandon, last_error: Option<&crate::Error>) { + let ctx = context::RedoAbandonGivenUp { + stream: format!("{:?}", abandon.stream), + vshard: abandon.stream.vshard(), + group_id: abandon.group_id, + term: abandon.term, + len: abandon.len, + last_error: last_error.map(ToString::to_string), + }; + let _ = Capture::new( + EventKind::Error, + "redo stream abandon given up: the stream stays open until its group's next term", + ) + .domain(&ctx) + .emit(); +} + +/// Report a refusal of `stream`'s `entry` (`chunk` or `final`) at log index +/// `log_index` of `group_id`, on a replica that dropped the group's streams +/// when it left the group. The other replicas can hold the stream and +/// install its redo. Called from the data-group apply loop, the one site +/// that holds the entry. +pub fn redo_stream_lost_here( + group_id: u64, + log_index: u64, + stream: RedoStreamId, + entry: &'static str, + refusal: &RedoChunkError, +) { + let ctx = context::RedoStreamLostHere { + group_id, + log_index, + stream: format!("{stream:?}"), + vshard: stream.vshard(), + entry, + refusal: refusal.to_string(), + }; + let _ = Capture::new( + EventKind::InvariantViolation, + "redo stream entry refused on a replica that dropped the stream: held until a snapshot \ + of the group installs", + ) + .domain(&ctx) + .error_chain(error_chain_of(refusal)) + .emit(); +} + +/// Report that the snapshot debt of data groups this node left did not +/// persist. Their streams stay, with their WAL floor holds, until a later +/// membership pass records it. Called from that pass, the one site that +/// writes the debt. +pub fn redo_snapshot_debt_not_recorded(error: &crate::Error) { + let ctx = context::RedoSnapshotDebtNotRecorded { + error: error.to_string(), + }; + let _ = Capture::new( + EventKind::Error, + "snapshot debt of left data groups did not persist: their redo streams stay held", + ) + .domain(&ctx) + .error_chain(error_chain_of(error)) + .emit(); +} diff --git a/nodedb/src/event/wal_replay_kind.rs b/nodedb/src/event/wal_replay_kind.rs index 212e2627d..0ad4f4b43 100644 --- a/nodedb/src/event/wal_replay_kind.rs +++ b/nodedb/src/event/wal_replay_kind.rs @@ -139,6 +139,9 @@ pub(super) fn row_kind(record_type: RecordType) -> Option { // emits no per-edge event, so replay rebuilds none. | RecordType::GraphEdgeCut // A snapshot install marker writes no row. - | RecordType::SnapshotInstalled => None, + | RecordType::SnapshotInstalled + // A redo chunk holds bytes of a redo stream. The stream's final + // record carries the rows and their events. + | RecordType::RedoChunk => None, } } diff --git a/nodedb/src/main_boot/post_open.rs b/nodedb/src/main_boot/post_open.rs index a33a68987..2ddc723fd 100644 --- a/nodedb/src/main_boot/post_open.rs +++ b/nodedb/src/main_boot/post_open.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: BUSL-1.1 //! Post-`SharedState::open` catalog steps: fire the catalog-open gate, -//! replay the surrogate WAL, read back the cut barriers, and bootstrap the -//! superuser credential. +//! replay the surrogate WAL, rebuild the open redo streams, read back the cut +//! barriers, and bootstrap the superuser credential. use std::sync::Arc; @@ -26,6 +26,16 @@ pub(crate) async fn run( // Replay surrogate WAL records into the in-memory registry. bootstrap::credentials::replay_surrogate_wal(shared, wal_records, replay_tombstones); + // Open chunked redo streams, before any data group applies an entry. + let streams = shared.redo_chunks.rebuild(wal_records)?; + if streams.open > 0 { + tracing::info!( + open = streams.open, + closed = streams.closed, + "chunked redo streams rebuilt from the WAL" + ); + } + // Cut barriers, for the apply loop's commit HLC floors. shared .pitr diff --git a/nodedb/src/types/snapshot.rs b/nodedb/src/types/snapshot.rs index f8a3a21c7..6220622a8 100644 --- a/nodedb/src/types/snapshot.rs +++ b/nodedb/src/types/snapshot.rs @@ -287,6 +287,14 @@ pub struct TenantDataSnapshot { #[msgpack(default)] #[serde(default)] pub group_calvin: Option, + + /// The open chunked redo streams of a Raft group snapshot's group. A + /// follower caught up by the snapshot never applies the chunks it covers, + /// so it holds them from here for each stream's final entry. Empty in a + /// backup and a per-core part. + #[msgpack(default)] + #[serde(default)] + pub group_redo_streams: Vec, } /// One `(partition, epoch, index, sequence)` position with its held row. diff --git a/nodedb/src/wal/manager/append_redo_chunk.rs b/nodedb/src/wal/manager/append_redo_chunk.rs new file mode 100644 index 000000000..10fafe9ad --- /dev/null +++ b/nodedb/src/wal/manager/append_redo_chunk.rs @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! WAL appends for chunked transaction redo streams. +//! +//! A chunk past the WAL record limit spans several `RedoChunk` pieces. Only +//! the last piece carries the chunk entry's apply key. The WAL is written in +//! order, so a durable last piece proves every earlier piece durable, and the +//! proposal ledger never counts a chunk applied whose bytes are partial. + +use nodedb_wal::record::RecordType; + +use super::appender::NO_APPLY_KEY; +use super::core::WalManager; +use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; +use crate::wal::redo::{RedoChunkHeader, RedoChunkPiece, RedoChunkRecord, RedoStreamId}; + +/// Room one piece record keeps for its header beside the chunk bytes. +const PIECE_HEADER_BYTES: usize = 256; + +/// The header fields of every record a stream appends. +#[derive(Debug, Clone, Copy)] +pub struct RedoChunkPlacement { + pub tenant_id: TenantId, + pub vshard_id: VShardId, + pub database_id: DatabaseId, +} + +/// The LSNs of the first and the last record one chunk appended. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ChunkLsns { + pub first: Lsn, + pub last: Lsn, +} + +impl WalManager { + /// Append chunk `header.index` of its stream, holding `bytes`. The last + /// piece carries `apply_key`, every other piece [`NO_APPLY_KEY`]. + pub fn append_redo_chunk( + &self, + apply_key: u64, + placement: RedoChunkPlacement, + header: RedoChunkHeader, + bytes: &[u8], + ) -> crate::Result { + let room = self + .max_payload() + .checked_sub(PIECE_HEADER_BYTES) + .filter(|room| *room > 0) + .ok_or_else(|| crate::Error::Internal { + detail: format!( + "WAL record limit {} leaves no room for a redo chunk piece", + self.max_payload() + ), + })?; + let parts: Vec<&[u8]> = if bytes.is_empty() { + vec![bytes] + } else { + bytes.chunks(room).collect() + }; + let pieces = u32::try_from(parts.len()).map_err(|_| crate::Error::Internal { + detail: format!("a redo chunk spans {} WAL pieces", parts.len()), + })?; + let mut first = None; + let mut last = Lsn::new(0); + for (piece, part) in (0u32..).zip(parts) { + let key = if piece + 1 == pieces { + apply_key + } else { + NO_APPLY_KEY + }; + let record = RedoChunkRecord::Piece(RedoChunkPiece { + header, + piece, + pieces, + bytes: part.to_vec(), + }); + last = self.appender(key).append_record( + RecordType::RedoChunk, + placement.tenant_id, + placement.vshard_id, + placement.database_id, + &record.to_bytes()?, + )?; + first.get_or_insert(last); + } + Ok(ChunkLsns { + first: first.unwrap_or(last), + last, + }) + } + + /// Append the close of `stream`: boot opens it no more. + pub fn append_redo_stream_closed( + &self, + apply_key: u64, + placement: RedoChunkPlacement, + stream: RedoStreamId, + ) -> crate::Result { + self.appender(apply_key).append_record( + RecordType::RedoChunk, + placement.tenant_id, + placement.vshard_id, + placement.database_id, + &RedoChunkRecord::Closed { stream }.to_bytes()?, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn placement() -> RedoChunkPlacement { + RedoChunkPlacement { + tenant_id: TenantId::new(1), + vshard_id: VShardId::new(4), + database_id: DatabaseId::DEFAULT, + } + } + + fn header() -> RedoChunkHeader { + RedoChunkHeader { + stream: RedoStreamId::Session { + vshard: 4, + idempotency_key: 77, + }, + group_id: 1, + term: 2, + len: 0, + index: 0, + } + } + + #[test] + fn a_chunk_past_the_record_limit_keys_only_its_last_piece() { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = WalManager::open_for_testing(&dir.path().join("wal")).expect("open wal"); + let bytes: Vec = (0..wal.max_payload() * 2).map(|i| i as u8).collect(); + let mut head = header(); + head.len = bytes.len() as u64; + let lsns = wal + .append_redo_chunk(0xC0FFEE, placement(), head, &bytes) + .expect("append"); + wal.sync().expect("sync"); + assert!(lsns.last > lsns.first); + + let records = wal.replay().expect("replay"); + let mut joined = Vec::new(); + let mut keys = Vec::new(); + for record in &records { + let RedoChunkRecord::Piece(piece) = + RedoChunkRecord::from_bytes(&record.payload).expect("decode") + else { + panic!("expected a piece"); + }; + assert_eq!(piece.pieces as usize, records.len()); + joined.extend_from_slice(&piece.bytes); + keys.push(record.apply_key()); + } + assert_eq!(joined, bytes); + let (last, rest) = keys.split_last().expect("pieces"); + assert_eq!(*last, 0xC0FFEE); + assert!(rest.iter().all(|key| *key == NO_APPLY_KEY)); + } +} diff --git a/nodedb/src/wal/manager/core.rs b/nodedb/src/wal/manager/core.rs index 0dd08b61a..03ad79e25 100644 --- a/nodedb/src/wal/manager/core.rs +++ b/nodedb/src/wal/manager/core.rs @@ -49,6 +49,8 @@ pub struct WalManager { /// Commit-time anchors of this WAL, stamped from the node's HLC. The writer /// records one per fsynced batch. Replay records the persisted ones. pub(super) time_anchors: Arc, + /// Floors truncation keeps for holders that read records back at boot. + pub(super) floor_holds: super::floor_holds::WalFloorHolds, } impl WalManager { @@ -76,6 +78,11 @@ impl WalManager { Ok(self.crdt_signing_root) } + /// The floors truncation of this WAL keeps. + pub fn floor_holds(&self) -> &super::floor_holds::WalFloorHolds { + &self.floor_holds + } + /// This WAL's commit-time anchors. pub fn time_anchors(&self) -> &Arc { &self.time_anchors @@ -197,6 +204,7 @@ impl WalManager { commit_lock: tokio::sync::Mutex::new(()), durable_notify: tokio::sync::Notify::new(), time_anchors, + floor_holds: super::floor_holds::WalFloorHolds::default(), }) } diff --git a/nodedb/src/wal/manager/floor_holds.rs b/nodedb/src/wal/manager/floor_holds.rs new file mode 100644 index 000000000..fbeb65122 --- /dev/null +++ b/nodedb/src/wal/manager/floor_holds.rs @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Floors WAL truncation must keep. +//! +//! A holder that reads records back after a restart takes a [`WalFloorHold`] +//! at the lowest LSN it needs. [`super::WalManager::truncate_before`] never +//! deletes a segment holding a record at or above the lowest held LSN. A +//! hold lasts until its guard drops. + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +use crate::types::Lsn; + +/// The held floors of one WAL. +#[derive(Default)] +pub struct WalFloorHolds { + inner: Arc, +} + +#[derive(Default)] +struct HoldsInner { + next_id: AtomicU64, + /// Hold id to the LSN it holds. + held: Mutex>, +} + +/// One held floor. Dropping it releases the floor. +pub struct WalFloorHold { + inner: Arc, + id: u64, + lsn: Lsn, +} + +impl std::fmt::Debug for WalFloorHold { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WalFloorHold") + .field("id", &self.id) + .field("lsn", &self.lsn) + .finish() + } +} + +impl WalFloorHolds { + /// Hold truncation below `lsn` until the returned guard drops. + pub fn hold(&self, lsn: Lsn) -> WalFloorHold { + let id = self.inner.next_id.fetch_add(1, Ordering::Relaxed); + self.inner + .held + .lock() + .unwrap_or_else(|p| p.into_inner()) + .insert(id, lsn.as_u64()); + WalFloorHold { + inner: Arc::clone(&self.inner), + id, + lsn, + } + } + + /// The lowest held LSN, or `None` when nothing is held. + pub fn lowest(&self) -> Option { + self.inner + .held + .lock() + .unwrap_or_else(|p| p.into_inner()) + .values() + .min() + .map(|lsn| Lsn::new(*lsn)) + } + + /// `lsn`, lowered to the lowest held LSN. + pub fn clamp(&self, lsn: Lsn) -> Lsn { + match self.lowest() { + Some(held) if held < lsn => held, + _ => lsn, + } + } +} + +impl WalFloorHold { + /// The LSN this guard holds. + pub fn lsn(&self) -> Lsn { + self.lsn + } +} + +impl Drop for WalFloorHold { + fn drop(&mut self) { + self.inner + .held + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(&self.id); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_lowest_live_hold_clamps_truncation() { + let holds = WalFloorHolds::default(); + assert_eq!(holds.clamp(Lsn::new(50)), Lsn::new(50)); + let high = holds.hold(Lsn::new(30)); + let low = holds.hold(Lsn::new(10)); + assert_eq!(holds.clamp(Lsn::new(50)), Lsn::new(10)); + assert_eq!(holds.clamp(Lsn::new(5)), Lsn::new(5)); + drop(low); + assert_eq!(holds.clamp(Lsn::new(50)), Lsn::new(30)); + drop(high); + assert_eq!(holds.lowest(), None); + } +} diff --git a/nodedb/src/wal/manager/mod.rs b/nodedb/src/wal/manager/mod.rs index 66227d851..2f83a3ef1 100644 --- a/nodedb/src/wal/manager/mod.rs +++ b/nodedb/src/wal/manager/mod.rs @@ -4,6 +4,7 @@ pub mod append; pub mod append_batch; pub mod append_index; pub mod append_metadata; +pub mod append_redo_chunk; pub mod append_restore_point; pub mod append_transaction; pub mod append_truncate; @@ -14,8 +15,11 @@ pub mod core; pub mod durable_commit; pub mod effect_stamp; pub mod encryption; +pub mod floor_holds; pub mod ops; pub mod replay; +pub use append_redo_chunk::{ChunkLsns, RedoChunkPlacement}; pub use appender::{AppendSink, NO_APPLY_KEY, RecordedAppend, WalAppender}; pub use core::WalManager; +pub use floor_holds::{WalFloorHold, WalFloorHolds}; diff --git a/nodedb/src/wal/manager/ops.rs b/nodedb/src/wal/manager/ops.rs index 3a1b0cff2..1d4295027 100644 --- a/nodedb/src/wal/manager/ops.rs +++ b/nodedb/src/wal/manager/ops.rs @@ -15,9 +15,18 @@ impl WalManager { &self, checkpoint_lsn: Lsn, ) -> crate::Result { + // A held floor keeps every record a holder reads back at boot. + let bound = self.floor_holds.clamp(checkpoint_lsn); + if bound < checkpoint_lsn { + tracing::debug!( + checkpoint_lsn = checkpoint_lsn.as_u64(), + held_lsn = bound.as_u64(), + "WAL truncation held below the checkpoint" + ); + } let wal = self.wal.lock().unwrap_or_else(|p| p.into_inner()); let result = wal - .truncate_before(checkpoint_lsn.as_u64()) + .truncate_before(bound.as_u64()) .map_err(crate::Error::Wal)?; if result.segments_deleted > 0 { diff --git a/nodedb/src/wal/mod.rs b/nodedb/src/wal/mod.rs index 229e71169..c9d77ce73 100644 --- a/nodedb/src/wal/mod.rs +++ b/nodedb/src/wal/mod.rs @@ -21,11 +21,12 @@ pub(crate) use crdt_payload::{ }; pub use manager::WalManager; pub use redo::{ - CalvinStamp, CapturedEntry, CascadedEdge, ContinuedRedo, CrossShardAppliedKey, EVERY_ROW, - EdgeCutRedo, EdgeDeleteRedo, EdgePutRedo, GroupMembership, NodeCascadeRedo, OriginAppend, - PublishPosition, RedoPublish, RedoRecord, RedoRowChange, RedoRowKind, RedoRowSource, - RedoSubRecord, RowSourceIndex, SplitRedo, WriteGroup, WriteGroupRecord, WriteSetCapture, - split_redo, + CalvinStamp, CapturedEntry, CarriedRedoStream, CascadedEdge, ContinuedRedo, + CrossShardAppliedKey, EVERY_ROW, EdgeCutRedo, EdgeDeleteRedo, EdgePutRedo, GroupMembership, + NodeCascadeRedo, OriginAppend, PublishPosition, RedoChunkHeader, RedoChunkPiece, + RedoChunkRecord, RedoPublish, RedoRecord, RedoRowChange, RedoRowKind, RedoRowSource, + RedoStreamId, RedoSubRecord, RowSourceIndex, SplitRedo, WriteGroup, WriteGroupRecord, + WriteSetCapture, split_redo, }; pub use replay::SyncHwmReplayMaps; pub use replay::SyncHwmReplayStats; diff --git a/nodedb/src/wal/redo/chunk.rs b/nodedb/src/wal/redo/chunk.rs new file mode 100644 index 000000000..7e0127d8e --- /dev/null +++ b/nodedb/src/wal/redo/chunk.rs @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Chunked transaction redo: the stream a redo's chunks share, and the WAL +//! record of one chunk. +//! +//! A committed redo past the redo entry limit travels through its data-group +//! log as chunk entries followed by a final entry. Every replica appends each +//! chunk it applies as `RedoChunk` records and holds the bytes until the final +//! entry assembles them. Boot rebuilds the open streams from these records. + +use serde::{Deserialize, Serialize}; + +/// The stream one chunked redo's chunks and final entry share. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub enum RedoStreamId { + /// A committed Calvin slice. Each proposal attempt of the position + /// opens its own stream. + Calvin { + vshard: u32, + epoch: u64, + position: u32, + attempt: u32, + }, + /// A session COMMIT. The final entry's idempotency key names the stream. + Session { vshard: u32, idempotency_key: u64 }, +} + +impl RedoStreamId { + /// The vShard whose data-group log carries the stream. + pub fn vshard(&self) -> u32 { + match self { + Self::Calvin { vshard, .. } | Self::Session { vshard, .. } => *vshard, + } + } +} + +/// Where one chunk sits in its stream. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct RedoChunkHeader { + pub stream: RedoStreamId, + /// The data group whose log carried the chunk. + pub group_id: u64, + /// The Raft term of the stream's first chunk entry. + pub term: u64, + /// The byte length of the whole stream, as every chunk declares it. + pub len: u64, + /// The chunk's place in the stream, from `0`. + pub index: u32, +} + +/// One WAL piece of one chunk. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct RedoChunkPiece { + pub header: RedoChunkHeader, + /// The piece's place in the chunk, from `0`. + pub piece: u32, + /// How many pieces the chunk spans. + pub pieces: u32, + pub bytes: Vec, +} + +/// The payload of a `RedoChunk` WAL record. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub enum RedoChunkRecord { + /// One piece of one chunk. + Piece(RedoChunkPiece), + /// The stream closed without its final entry. Boot opens it no more. + Closed { stream: RedoStreamId }, +} + +impl RedoChunkRecord { + pub fn to_bytes(&self) -> crate::Result> { + zerompk::to_msgpack_vec(self).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("redo chunk record encode: {e}"), + }) + } + + pub fn from_bytes(bytes: &[u8]) -> crate::Result { + zerompk::from_msgpack(bytes).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("redo chunk record decode: {e}"), + }) + } +} + +/// One open stream a data-group snapshot carries to the follower that +/// installs it. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CarriedRedoStream { + pub stream: RedoStreamId, + pub term: u64, + pub len: u64, + /// The chunks the stream holds, in index order. + pub chunks: Vec>, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_chunk_record_round_trips() { + let record = RedoChunkRecord::Piece(RedoChunkPiece { + header: RedoChunkHeader { + stream: RedoStreamId::Session { + vshard: 3, + idempotency_key: 0xAB, + }, + group_id: 2, + term: 5, + len: 10, + index: 1, + }, + piece: 0, + pieces: 2, + bytes: vec![1, 2, 3], + }); + let bytes = record.to_bytes().expect("encode"); + assert_eq!(RedoChunkRecord::from_bytes(&bytes).expect("decode"), record); + let closed = RedoChunkRecord::Closed { + stream: RedoStreamId::Calvin { + vshard: 1, + epoch: 9, + position: 4, + attempt: 2, + }, + }; + let bytes = closed.to_bytes().expect("encode"); + assert_eq!(RedoChunkRecord::from_bytes(&bytes).expect("decode"), closed); + } +} diff --git a/nodedb/src/wal/redo/mod.rs b/nodedb/src/wal/redo/mod.rs index 6db7d70a2..685e575d4 100644 --- a/nodedb/src/wal/redo/mod.rs +++ b/nodedb/src/wal/redo/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 pub mod capture; +pub mod chunk; pub mod continuation; pub mod group; pub mod record; @@ -9,6 +10,9 @@ pub mod row_changes; pub mod row_sources; pub use capture::{CapturedEffect, CapturedEntry, OriginAppend, WriteSetCapture}; +pub use chunk::{ + CarriedRedoStream, RedoChunkHeader, RedoChunkPiece, RedoChunkRecord, RedoStreamId, +}; pub use continuation::{ContinuedRedo, SplitRedo, split_redo}; pub use group::{CascadedEdge, GroupMembership, NodeCascadeRedo, WriteGroup, WriteGroupRecord}; pub use record::{ diff --git a/nodedb/src/wal/replay/surrogate/dispatch.rs b/nodedb/src/wal/replay/surrogate/dispatch.rs index 23c6eba88..8f32809f6 100644 --- a/nodedb/src/wal/replay/surrogate/dispatch.rs +++ b/nodedb/src/wal/replay/surrogate/dispatch.rs @@ -129,7 +129,10 @@ pub fn replay_surrogate_records( | RecordType::SnapshotInstalled // A write group carries row images, whose surrogates were bound // before the write that stored them. - | RecordType::WriteGroup => {} + | RecordType::WriteGroup + // A redo chunk holds stream bytes; its final record binds nothing + // the carried identities did not bind first. + | RecordType::RedoChunk => {} } } Ok(stats) diff --git a/nodedb/src/wal/replay/sync_hwm/advance.rs b/nodedb/src/wal/replay/sync_hwm/advance.rs index 34dc15918..4352b3ca9 100644 --- a/nodedb/src/wal/replay/sync_hwm/advance.rs +++ b/nodedb/src/wal/replay/sync_hwm/advance.rs @@ -137,7 +137,8 @@ pub fn replay_sync_hwm_records( | RecordType::GraphNodeCascade | RecordType::GraphEdgeCut | RecordType::SnapshotInstalled - | RecordType::WriteGroup => {} + | RecordType::WriteGroup + | RecordType::RedoChunk => {} } } diff --git a/nodedb/tests/crash_redo_chunk_stream.rs b/nodedb/tests/crash_redo_chunk_stream.rs new file mode 100644 index 000000000..2eb760a61 --- /dev/null +++ b/nodedb/tests/crash_redo_chunk_stream.rs @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A chunked session redo survives a crash inside its stream. +//! +//! The server boots with `tuning.calvin.max_redo_entry_bytes` at its floor, +//! so a commit of a few hundred kilobytes travels as chunk entries and a +//! final entry. Each test kills the server at a fail point inside the stream +//! and restarts it on the same data directory: +//! +//! - after every chunk is durable and before the final entry applies: boot +//! rebuilds the stream from its WAL records, and the re-delivered final +//! entry installs every row once; +//! - after the first chunk is durable: no final entry exists, so no row +//! installs, and the stream drops at the group's next term. A later +//! chunked commit installs as usual. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +mod crash_harness; + +use std::time::Duration; + +use crash_harness::{CrashHarness, diagnostics}; + +/// The redo entry limit the server boots with: the smallest the config takes. +const ENTRY_LIMIT: usize = 64 * 1024; + +/// Rows one statement inserts. +const ROWS_PER_STATEMENT: usize = 100; + +/// Statements one transaction runs. With 400-byte values the redo holds +/// several times `ENTRY_LIMIT`. +const STATEMENTS: usize = 6; + +/// Bounded wait for the injected abort. +const CRASH_TIMEOUT: Duration = Duration::from_secs(60); + +/// A harness whose server reads a config file with the lowered entry limit. +fn harness() -> CrashHarness { + let h = CrashHarness::new(); + let config = h.data_dir().join("redo_chunk.toml"); + std::fs::write( + &config, + format!("[tuning.calvin]\nmax_redo_entry_bytes = {ENTRY_LIMIT}\n"), + ) + .expect("write config file"); + let path = config.to_string_lossy().to_string(); + h.with_env("NODEDB_CONFIG", &path) +} + +async fn count(h: &CrashHarness, coll: &str) -> usize { + let rows = h + .query_col_idx(&format!("SELECT COUNT(*) FROM {coll}"), 0) + .await; + assert_eq!(rows.len(), 1, "expected one COUNT(*) row, got {rows:?}"); + rows[0].parse().expect("COUNT(*) is a number") +} + +/// Run one large transaction on its own connection. Returns whether COMMIT +/// answered success: a server that aborts mid-commit answers nothing. +async fn commit_large_transaction(h: &CrashHarness, coll: &str, prefix: &str) -> bool { + let (client, connection) = tokio_postgres::connect(&h.pgwire_conn_str(), tokio_postgres::NoTls) + .await + .expect("connect"); + let conn = tokio::spawn(async move { + let _ = connection.await; + }); + let filler = "x".repeat(400); + client.simple_query("BEGIN").await.expect("BEGIN"); + for statement in 0..STATEMENTS { + let values: Vec = (0..ROWS_PER_STATEMENT) + .map(|row| { + let id = statement * ROWS_PER_STATEMENT + row; + format!("('{prefix}{id}', '{filler}')") + }) + .collect(); + client + .simple_query(&format!( + "INSERT INTO {coll} (id, value) VALUES {}", + values.join(", ") + )) + .await + .expect("INSERT inside the transaction"); + } + let committed = client.simple_query("COMMIT").await.is_ok(); + drop(client); + let _ = conn.await; + committed +} + +/// Arm `fail_point`, crash the server inside a chunked commit, and restart it +/// without the fail point. +async fn crash_inside_a_chunked_commit(h: &mut CrashHarness, fail_point: &str) { + h.set_env("NODEDB_FAILPOINTS", &format!("{fail_point}=abort")); + h.spawn(); + h.wait_ready(); + h.exec( + "CREATE COLLECTION chunk_crash (id STRING PRIMARY KEY, value STRING) \ + WITH (engine='document_strict')", + ) + .await; + let committed = commit_large_transaction(h, "chunk_crash", "a").await; + assert!( + !committed, + "COMMIT answered success, but the server was armed to abort at `{fail_point}`" + ); + h.await_self_crash(CRASH_TIMEOUT); + let log = h.server_log(); + assert!( + log.contains(&format!("fail_point aborting process: {fail_point}")), + "the server exited, but not at the armed fail point `{fail_point}`, so this test \ + proves nothing about a crash inside a redo stream.{}\n{}", + h.keep_data_dir_note(), + diagnostics::log_tail_section(&log) + ); + h.clear_env("NODEDB_FAILPOINTS"); + h.reopen(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_crash_before_the_final_entry_applies_installs_the_whole_redo_once() { + let mut h = harness(); + crash_inside_a_chunked_commit(&mut h, "redo_chunk::before_final_install").await; + let expected = STATEMENTS * ROWS_PER_STATEMENT; + assert_eq!( + count(&h, "chunk_crash").await, + expected, + "the committed final entry must install every row of its rebuilt stream once\n{}", + diagnostics::log_tail_section(&h.server_log()) + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_crash_between_chunk_entries_installs_nothing_and_frees_the_stream() { + let mut h = harness(); + crash_inside_a_chunked_commit(&mut h, "redo_chunk::after_chunk_durable").await; + assert_eq!( + count(&h, "chunk_crash").await, + 0, + "a stream with no final entry must install no row" + ); + assert!( + commit_large_transaction(&h, "chunk_crash", "b").await, + "a chunked commit after the crash must succeed\n{}", + diagnostics::log_tail_section(&h.server_log()) + ); + assert_eq!( + count(&h, "chunk_crash").await, + STATEMENTS * ROWS_PER_STATEMENT + ); +} diff --git a/nodedb/tests/inproc/cases/calvin_executor_apply.rs b/nodedb/tests/inproc/cases/calvin_executor_apply.rs index a86e88c66..d622399e8 100644 --- a/nodedb/tests/inproc/cases/calvin_executor_apply.rs +++ b/nodedb/tests/inproc/cases/calvin_executor_apply.rs @@ -146,7 +146,9 @@ fn stage_and_resolve( rx, PhysicalPlan::Meta(MetaOp::CalvinResolve { epoch, position: 0 }), ); - RedoRecord::from_bytes(&redo).expect("decode resolved redo") + let answer: nodedb_physical::physical_plan::CalvinResolved = + zerompk::from_msgpack(&redo).expect("decode resolved answer"); + RedoRecord::from_bytes(&answer.redo).expect("decode resolved redo") } /// Flush `redo` as the Calvin transaction at `(epoch, 0)` with its record at diff --git a/nodedb/tests/inproc/cases/calvin_executor_panic_recovery.rs b/nodedb/tests/inproc/cases/calvin_executor_panic_recovery.rs index 4329149c8..9c1d3cc34 100644 --- a/nodedb/tests/inproc/cases/calvin_executor_panic_recovery.rs +++ b/nodedb/tests/inproc/cases/calvin_executor_panic_recovery.rs @@ -104,7 +104,11 @@ fn stage_then_flush( let mut request = make_request(PhysicalPlan::Meta(MetaOp::CalvinFlush { epoch, position: 0, - redo: resolved.payload.to_vec(), + redo: zerompk::from_msgpack::( + resolved.payload.as_bytes(), + ) + .expect("decode resolved answer") + .redo, collections: Vec::new(), sum_targets: Vec::new(), })); diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply.rs index 3fc15f168..5eee3f8bc 100644 --- a/nodedb/tests/inproc/cases/calvin_two_phase_apply.rs +++ b/nodedb/tests/inproc/cases/calvin_two_phase_apply.rs @@ -113,12 +113,19 @@ fn resolved_flush( PhysicalPlan::Meta(MetaOp::CalvinFlush { epoch, position: 0, - redo: resolved.payload.to_vec(), + redo: resolved_redo(&resolved), collections: Vec::new(), sum_targets: Vec::new(), }) } +/// The redo record a `CalvinResolve` answered with. +fn resolved_redo(resolved: &Response) -> Vec { + let answer: nodedb_physical::physical_plan::CalvinResolved = + zerompk::from_msgpack(resolved.payload.as_bytes()).expect("decode resolved answer"); + answer.redo +} + /// A write committed through the Calvin path to seed a write version. struct CalvinSeed<'a> { epoch: u64, @@ -175,7 +182,7 @@ fn commit_calvin( PhysicalPlan::Meta(MetaOp::CalvinFlush { epoch: seed.epoch, position: 0, - redo: resolved.payload.to_vec(), + redo: resolved_redo(&resolved), collections: vec![seed.collection.to_string()], sum_targets: Vec::new(), }), diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index 5696b97e4..4c9118e37 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -169,6 +169,7 @@ mod query_function_engine_gate; mod quota_bitemporal_composition; mod quota_sieve_routing_composition; mod redaction_policy_database_scope; +mod redo_chunked_session_commit; mod reindex_concurrent; mod restart_refused_write_not_resurrected; mod rls_policy_database_scope; diff --git a/nodedb/tests/wire/cases/redo_chunked_session_commit.rs b/nodedb/tests/wire/cases/redo_chunked_session_commit.rs new file mode 100644 index 000000000..2ffc1b04a --- /dev/null +++ b/nodedb/tests/wire/cases/redo_chunked_session_commit.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A session COMMIT whose redo passes `tuning.calvin.max_redo_entry_bytes` +//! travels through its data-group log as chunk entries and a final entry. +//! The commit installs every row once, live and after a WAL-only restart. + +use crate::harness::TestServer; + +/// The redo entry limit the server boots with: the smallest the config takes. +const ENTRY_LIMIT: usize = 64 * 1024; + +/// Rows one statement inserts. +const ROWS_PER_STATEMENT: usize = 100; + +/// Statements one transaction runs. With 400-byte values the redo holds +/// several times `ENTRY_LIMIT`. +const STATEMENTS: usize = 6; + +async fn count(srv: &TestServer, coll: &str) -> usize { + let rows = srv + .query_text(&format!("SELECT COUNT(*) FROM {coll}")) + .await + .unwrap(); + rows[0].parse().expect("COUNT(*) is a number") +} + +/// Insert `STATEMENTS * ROWS_PER_STATEMENT` rows with ids from `prefix` in +/// one transaction. +async fn commit_large_transaction(srv: &TestServer, coll: &str, prefix: &str) { + let filler = "x".repeat(400); + srv.exec("BEGIN").await.unwrap(); + for statement in 0..STATEMENTS { + let values: Vec = (0..ROWS_PER_STATEMENT) + .map(|row| { + let id = statement * ROWS_PER_STATEMENT + row; + format!("('{prefix}{id}', '{filler}')") + }) + .collect(); + srv.exec(&format!( + "INSERT INTO {coll} (id, value) VALUES {}", + values.join(", ") + )) + .await + .unwrap(); + } + srv.exec("COMMIT").await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_chunked_session_commit_installs_once() { + let srv = TestServer::start_with_max_redo_entry_bytes(ENTRY_LIMIT).await; + srv.exec( + "CREATE COLLECTION chunked_commit (id STRING PRIMARY KEY, value STRING) \ + WITH (engine='document_strict')", + ) + .await + .unwrap(); + + commit_large_transaction(&srv, "chunked_commit", "a").await; + let expected = STATEMENTS * ROWS_PER_STATEMENT; + assert_eq!( + count(&srv, "chunked_commit").await, + expected, + "a chunked commit must install every row exactly once" + ); + + // The finished stream frees its bytes and its floor: a second large + // commit chunks and installs too. + commit_large_transaction(&srv, "chunked_commit", "b").await; + assert_eq!(count(&srv, "chunked_commit").await, 2 * expected); + + let (srv, dir) = srv.take_dir(); + srv.graceful_shutdown().await; + let (srv, _dir) = TestServer::open_on_path_with_max_redo_entry_bytes(dir, ENTRY_LIMIT).await; + assert_eq!( + count(&srv, "chunked_commit").await, + 2 * expected, + "WAL replay of a chunked commit must restore every row exactly once" + ); +} diff --git a/nodedb/tests/wire/harness/config_toml.rs b/nodedb/tests/wire/harness/config_toml.rs index 64425e0ca..86f860b40 100644 --- a/nodedb/tests/wire/harness/config_toml.rs +++ b/nodedb/tests/wire/harness/config_toml.rs @@ -49,6 +49,9 @@ pub(super) struct TuningOverrides { /// Overrides `[tuning.vector] seal_threshold` so a test can observe HNSW /// segment builds on a few hundred vectors. pub(super) vector_seal_threshold: Option, + /// Overrides `[tuning.calvin] max_redo_entry_bytes` so a test can drive + /// chunked redo with a commit of a few hundred kilobytes. + pub(super) max_redo_entry_bytes: Option, } impl TuningOverrides { @@ -89,6 +92,14 @@ impl TuningOverrides { } } + /// Boot with a lowered redo entry limit. + pub(super) fn max_redo_entry_bytes(bytes: usize) -> Self { + Self { + max_redo_entry_bytes: Some(bytes), + ..Self::default() + } + } + /// Boot with a lowered vector seal threshold. pub(super) fn vector_seal_threshold(vectors: usize) -> Self { Self { @@ -141,6 +152,11 @@ pub(super) fn write_config(dir: &Path, auth_mode: AuthMode, tuning: TuningOverri if let Some(vectors) = tuning.vector_seal_threshold { toml.push_str(&format!("\n[tuning.vector]\nseal_threshold = {vectors}\n")); } + if let Some(bytes) = tuning.max_redo_entry_bytes { + toml.push_str(&format!( + "\n[tuning.calvin]\nmax_redo_entry_bytes = {bytes}\n" + )); + } toml.push_str(&format!( "\n[backup_encryption]\nkey_path = {}\n", toml_quote(&write_backup_kek(dir)) diff --git a/nodedb/tests/wire/harness/lifecycle.rs b/nodedb/tests/wire/harness/lifecycle.rs index ae661f291..799837cf4 100644 --- a/nodedb/tests/wire/harness/lifecycle.rs +++ b/nodedb/tests/wire/harness/lifecycle.rs @@ -162,6 +162,36 @@ impl TestServer { (server, dir) } + /// Spawn a single-core NodeDB server with a lowered redo entry limit, so + /// a commit of a few hundred kilobytes travels as a chunked redo. + pub async fn start_with_max_redo_entry_bytes(bytes: usize) -> Self { + let dir = tempfile::tempdir().expect("tempdir"); + let spawned = process::spawn( + dir.path(), + AuthMode::Trust, + TuningOverrides::max_redo_entry_bytes(bytes), + 1, + ); + Self::connect_and_build(spawned, dir, AuthMode::Trust).await + } + + /// Reopen `dir` with the lowered redo entry limit a server started by + /// [`Self::start_with_max_redo_entry_bytes`] used. + pub async fn open_on_path_with_max_redo_entry_bytes( + dir: TestDataDir, + bytes: usize, + ) -> (Self, TestDataDir) { + let spawned = process::spawn( + dir.path(), + AuthMode::Trust, + TuningOverrides::max_redo_entry_bytes(bytes), + 1, + ); + let placeholder = tempfile::tempdir().expect("placeholder tempdir"); + let server = Self::connect_and_build(spawned, placeholder, AuthMode::Trust).await; + (server, dir) + } + /// Open a server backed by an existing data directory, reopened in place /// so a previous server's data is visible after boot. `dir` is not /// consumed — ownership stays with the caller. From a4b4970660eefe7c7ebf2e0624637d278eef3bde Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sat, 10 Oct 2026 04:54:04 +0800 Subject: [PATCH 05/19] feat(calvin): replicate committed slices through the data group Leader-driven commit - Only the vShard's data-group leader stages, votes, resolves, and proposes a committed slice's stamped redo. Other replicas hold each granted txn as Following and install its slice from the log. - Promotion stages every Following txn again in sequencer order. Demotion discards the leader's staged state and owed votes. - A committed redo is proposed right after the resolve. The apply loop claims each position in the applied ledger, so a copy installs at most once. A slice that reads a whole collection at resolve waits for its turn behind lower txns. Apply notifications - The apply loop pushes one event per concluded redo into a bounded per-vShard scheduler inbox. The scheduler deposits the reply, owes the CompletionAck, releases locks, and completes. A refused or unapplied redo halts the scheduler. - The separate flush turn, write-version record, redo window, and Data Plane fence are removed in favour of this path. A drop discards a txn's staged state on its core. Transport - An inbound RPC stream whose handler started but never answered is now reset instead of finished, so a sender can tell an unknown outcome from a refusal before the handler and never resends the former. Tests cover leader change mid-transaction, write-version parity, and the reworked two-phase apply cases. --- .../cases/calvin_leader_change_mid_txn.rs | 124 +++ .../tests/common_suite/cases/mod.rs | 1 + .../cases/single_node_calvin_two_phase.rs | 18 +- .../single_node_calvin_write_versions.rs | 11 +- nodedb-cluster/src/multi_raft/peer_contact.rs | 9 + nodedb-cluster/src/multi_raft/proposals.rs | 6 +- .../src/rpc_codec/data_plane_error.rs | 6 + .../src/transport/client/attempt.rs | 3 +- .../src/transport/client/transport.rs | 41 +- nodedb-cluster/src/transport/mod.rs | 1 + nodedb-cluster/src/transport/reply_stream.rs | 70 ++ nodedb-cluster/src/transport/server.rs | 17 +- .../src/physical_plan/calvin_reply.rs | 3 + nodedb-physical/src/physical_plan/meta.rs | 49 +- nodedb-raft/src/node/core.rs | 14 + nodedb-raft/src/node/internal.rs | 22 + nodedb/src/bootstrap/calvin_catch_up.rs | 6 +- nodedb/src/bridge/envelope/error_code.rs | 4 + nodedb/src/control/change_stream/mod.rs | 2 +- .../src/control/change_stream/stream/mod.rs | 2 +- .../scheduler/driver/core/apply_result.rs | 211 ++++ .../calvin/scheduler/driver/core/catch_up.rs | 3 + .../scheduler/driver/core/commit_redo.rs | 363 +++---- .../driver/core/commit_resolution_dispatch.rs | 104 -- .../driver/core/commit_resolve/apply_tail.rs | 674 ------------- .../driver/core/commit_resolve/drop_tail.rs | 90 ++ .../driver/core/commit_resolve/flush_parts.rs | 28 - .../driver/core/commit_resolve/mod.rs | 21 +- .../driver/core/commit_resolve/verdict.rs | 304 +++--- .../driver/core/commit_resolve/vote.rs | 134 ++- .../scheduler/driver/core/completion_route.rs | 148 ++- .../scheduler/driver/core/cut_marker.rs | 103 +- .../calvin/scheduler/driver/core/deferred.rs | 168 ++-- .../driver/core/dispatch/active_dispatch.rs | 47 +- .../driver/core/dispatch/bind_identities.rs | 31 +- .../scheduler/driver/core/dispatch/follow.rs | 84 ++ .../driver/core/dispatch/incarnation.rs | 77 +- .../scheduler/driver/core/dispatch/mod.rs | 4 +- .../driver/core/dispatch/primary_write.rs | 52 +- .../driver/core/dispatch/static_dispatch.rs | 73 +- .../driver/core/dispatch/unstaged.rs | 46 +- .../scheduler/driver/core/drop_dispatch.rs | 40 + .../scheduler/driver/core/flush_turn.rs | 225 ----- .../calvin/scheduler/driver/core/halt.rs | 80 +- .../scheduler/driver/core/install_gate.rs | 171 +--- .../calvin/scheduler/driver/core/intake.rs | 12 +- .../calvin/scheduler/driver/core/mod.rs | 21 +- .../scheduler/driver/core/owed/retry.rs | 46 +- .../calvin/scheduler/driver/core/parts.rs | 27 +- .../scheduler/driver/core/parts_lane.rs | 5 +- .../calvin/scheduler/driver/core/process.rs | 94 +- .../scheduler/driver/core/read_result.rs | 9 +- .../scheduler/driver/core/redo_applied.rs | 310 ++++++ .../scheduler/driver/core/redo_propose.rs | 450 +++++++++ .../scheduler/driver/core/redo_window.rs | 124 --- .../calvin/scheduler/driver/core/request.rs | 27 +- .../scheduler/driver/core/resolve_turn.rs | 126 +++ .../calvin/scheduler/driver/core/role.rs | 438 +++++++++ .../calvin/scheduler/driver/core/run_loop.rs | 42 +- .../calvin/scheduler/driver/core/scheduler.rs | 30 +- .../scheduler/driver/core/staged_vote.rs | 30 +- .../scheduler/driver/core/test_proposer.rs | 57 +- .../scheduler/driver/core/test_support.rs | 43 +- .../driver/core/write_version_record.rs | 306 ------ .../cluster/calvin/scheduler/driver/types.rs | 191 ++-- .../control/cluster/calvin/scheduler/inbox.rs | 290 ++++++ .../cluster/calvin/scheduler/metrics.rs | 24 +- .../control/cluster/calvin/scheduler/mod.rs | 2 + .../cluster/calvin/scheduler/recovery.rs | 2 - .../control/cluster/data_plane_error_wire.rs | 18 + nodedb/src/control/cluster/sequencer_halt.rs | 6 +- .../src/control/cluster/start_raft/hooks.rs | 11 + .../apply_loop/calvin_redo.rs | 375 ++++++++ .../distributed_applier/apply_loop/lane.rs | 72 +- .../distributed_applier/apply_loop/mod.rs | 3 + .../apply_loop/redo_chunk.rs | 50 +- .../apply_loop/transaction_redo.rs | 156 ++- nodedb/src/control/fail_gate.rs | 37 +- .../error_map/class_parity/code_samples.rs | 7 +- .../src/control/planner/rls_injection/meta.rs | 1 - .../rls_injection/permission_tree/meta.rs | 1 - .../security/identity/plan_permission.rs | 1 - .../dispatch_utils/change_events/mod.rs | 5 +- .../dispatch_utils/change_events/publish.rs | 71 +- .../server/dispatch_utils/minted/records.rs | 5 - .../src/control/server/dispatch_utils/mod.rs | 5 +- .../server/dispatch_utils/write_abort.rs | 8 +- .../server/exchange/all_cores/dispatch.rs | 1 - .../authorization/requirements/collect.rs | 2 - .../src/control/server/shared/ddl/sqlstate.rs | 7 + .../control/server/shared/returning/inject.rs | 1 - .../shared/write_admission/admission_keys.rs | 1 - .../shared/write_admission/leader_gate.rs | 6 +- .../predicate/txn_buffering/classify.rs | 7 - .../predicate/user_data_write.rs | 40 +- .../sync/async_dispatch/delta/compensation.rs | 1 + nodedb/src/control/server/sync/refusal.rs | 3 + nodedb/src/control/state/calvin_counters.rs | 12 +- nodedb/src/control/state/calvin_local.rs | 6 +- .../decode/transaction_redo.rs | 53 +- .../src/control/wal_replication/encode/mod.rs | 5 +- .../encode/transaction_redo.rs | 164 +++- .../transaction_redo/chunks/store.rs | 51 + .../wal_replication/transaction_redo/mod.rs | 2 +- .../transaction_redo/payload.rs | 41 +- .../wal_replication/types/redo_chunk_wire.rs | 11 - .../wal_replication/types/replicated_write.rs | 8 +- .../types/transaction_redo_wire.rs | 9 +- .../data/executor/core_loop/calvin_fence.rs | 371 ------- .../data/executor/core_loop/calvin_state.rs | 33 +- .../data/executor/core_loop/commit_pending.rs | 8 +- .../src/data/executor/core_loop/fail_stop.rs | 10 +- nodedb/src/data/executor/core_loop/mod.rs | 1 - nodedb/src/data/executor/core_loop/tick.rs | 9 +- .../data/executor/core_loop/write_index.rs | 12 + nodedb/src/data/executor/dispatch/meta.rs | 14 - .../materialized_sum/divergence.rs | 9 +- .../executor/handlers/bulk_dml/admission.rs | 16 +- .../handlers/control/calvin/active_passive.rs | 12 +- .../handlers/control/calvin/discard.rs | 10 +- .../executor/handlers/control/calvin/flush.rs | 125 --- .../executor/handlers/control/calvin/mod.rs | 11 +- .../handlers/control/calvin/shared.rs | 4 - .../handlers/control/calvin/static_stage.rs | 20 +- .../handlers/control/calvin/test_commit.rs | 1 - .../handlers/control/calvin_active_verify.rs | 7 +- .../handlers/control/calvin_resolve.rs | 5 +- .../src/data/executor/handlers/control/mod.rs | 1 + .../handlers/control/reindex/waiter.rs | 7 +- .../handlers/point/apply_put/types.rs | 1 + .../snapshot/restore/tenant_snapshot.rs | 3 + .../data/executor/handlers/transaction/mod.rs | 4 + .../redo_apply/calvin_fold_tests.rs | 96 +- .../transaction/redo_apply/calvin_install.rs | 11 +- .../redo_apply/calvin_install_tests.rs | 2 +- .../transaction/redo_apply/document.rs | 22 +- .../handlers/transaction/redo_apply/mod.rs | 4 +- .../handlers/transaction/redo_apply/state.rs | 38 - .../handlers/transaction/write_version.rs | 8 +- .../handlers/transaction/write_version_kv.rs | 6 +- .../transaction/write_version_parity_tests.rs | 330 +++++++ .../data/executor/vector_checkpoint/write.rs | 1 + .../data/executor/wal_replay_redo_document.rs | 27 +- nodedb/src/data/executor/wal_replay_vector.rs | 1 + .../data/executor/wal_replay_vector_delete.rs | 1 + .../data/executor/wal_replay_vector_direct.rs | 3 + .../executor/wal_replay_vector_extended.rs | 3 + .../executor/wal_replay_vector_resolved.rs | 1 + .../data/executor/wal_replay_vector_sparse.rs | 2 + .../data/executor/wal_replay_vector_task.rs | 18 +- nodedb/src/diag/context/data_plane.rs | 4 +- nodedb/src/diag/context/mod.rs | 2 +- nodedb/src/diag/context/raft_apply.rs | 37 - nodedb/src/diag/mod.rs | 13 +- nodedb/src/diag/recording/mod.rs | 4 +- nodedb/src/diag/recording/raft_apply.rs | 29 - nodedb/src/error_from_data_plane.rs | 6 + nodedb/src/wal/redo/record.rs | 18 +- nodedb/src/wal/redo/replay.rs | 49 +- nodedb/tests/calvin_hold_liveness.rs | 34 +- nodedb/tests/crash_replay_stamp_calvin.rs | 116 +-- .../inproc/cases/calvin_executor_apply.rs | 58 +- .../cases/calvin_executor_panic_recovery.rs | 139 ++- .../calvin_executor_static_multishard.rs | 2 - .../inproc/cases/calvin_two_phase_apply.rs | 908 ------------------ .../cases/calvin_two_phase_apply/install.rs | 147 +++ .../cases/calvin_two_phase_apply/mod.rs | 19 + .../cases/calvin_two_phase_apply/phantom.rs | 236 +++++ .../cases/calvin_two_phase_apply/read_set.rs | 313 ++++++ .../cases/calvin_two_phase_apply/support.rs | 263 +++++ .../tests/wire/cases/sql_backup_calvin_cut.rs | 29 +- .../cases/sql_backup_restore_durable_marks.rs | 2 +- 172 files changed, 6337 insertions(+), 5018 deletions(-) create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_leader_change_mid_txn.rs create mode 100644 nodedb-cluster/src/transport/reply_stream.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/apply_result.rs delete mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolution_dispatch.rs delete mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/apply_tail.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/drop_tail.rs delete mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/flush_parts.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/follow.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/drop_dispatch.rs delete mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/flush_turn.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_applied.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs delete mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_window.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/resolve_turn.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/role.rs delete mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/write_version_record.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/inbox.rs create mode 100644 nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs delete mode 100644 nodedb/src/data/executor/core_loop/calvin_fence.rs create mode 100644 nodedb/src/data/executor/handlers/transaction/write_version_parity_tests.rs delete mode 100644 nodedb/tests/inproc/cases/calvin_two_phase_apply.rs create mode 100644 nodedb/tests/inproc/cases/calvin_two_phase_apply/install.rs create mode 100644 nodedb/tests/inproc/cases/calvin_two_phase_apply/mod.rs create mode 100644 nodedb/tests/inproc/cases/calvin_two_phase_apply/phantom.rs create mode 100644 nodedb/tests/inproc/cases/calvin_two_phase_apply/read_set.rs create mode 100644 nodedb/tests/inproc/cases/calvin_two_phase_apply/support.rs diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_leader_change_mid_txn.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_leader_change_mid_txn.rs new file mode 100644 index 000000000..b39473740 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_leader_change_mid_txn.rs @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A Calvin transaction whose data-group leader dies between its vote and +//! its redo commits once on every replica. +//! +//! 1. A 3-node cluster holds two collections on distinct vShards. A session +//! on a node that does not lead `a`'s data group runs +//! `BEGIN; INSERT a; INSERT b; COMMIT` in strict cross-shard mode. +//! 2. The fail gate `calvin::before_redo_propose::` holds `a`'s slice +//! after its vote and resolve, before its leader proposes the redo. +//! 3. The leader of `a`'s data group is killed while the redo is held. +//! 4. Released, the new leader stages the held txn again, resolves it, and +//! proposes its redo. Each collection holds its row exactly once on every +//! surviving node. +//! +//! The session's node submits the txn to the sequencer leader and waits for +//! its answer. When the victim also leads the sequencer group, that answer +//! is lost with it: the COMMIT reports an unknown outcome (`57014`) and the +//! txn still commits once. Otherwise the COMMIT succeeds. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::time::Duration; + +use nodedb_types::fail_point::{FailAction, FailGuard}; + +use super::calvin_multishard_fixture::{Fixture, keyed_ddl}; +use super::vshard_names::distinct_vshard_collections; +use crate::common::cluster_harness::shared_steps::{kill_node, leader_index_of, leader_of}; +use crate::common::cluster_harness::{TestClusterNode, wait_for}; + +/// How long the test waits for the leader to hold the redo. +const HOLD_DEADLINE: Duration = Duration::from_secs(15); + +/// A fresh session on `node` in strict cross-shard mode. +async fn strict_session(node: &TestClusterNode) -> tokio_postgres::Client { + let (client, connection) = tokio_postgres::connect( + &format!( + "host=127.0.0.1 port={} user=nodedb dbname=default", + node.pg_addr.port() + ), + tokio_postgres::NoTls, + ) + .await + .expect("connect a session"); + tokio::spawn(async move { + let _ = connection.await; + }); + client + .simple_query("SET cross_shard_txn = 'strict'") + .await + .expect("SET cross_shard_txn = strict"); + client +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_txn_whose_leader_dies_between_vote_and_redo_commits_once() { + let (col_a, col_b) = + distinct_vshard_collections("calvin_leader_change_0", "calvin_leader_change"); + let mut fx = Fixture::spawn(&[keyed_ddl(&col_a), keyed_ddl(&col_b)]).await; + fx.wait_group_mounted(&col_a).await; + fx.wait_group_mounted(&col_b).await; + + let victim = leader_index_of(&fx.cluster, &col_a); + let session_node = (0..fx.cluster.nodes.len()) + .find(|idx| *idx != victim) + .expect("a 3-node cluster has a node besides the leader"); + let victim_leads_sequencer = leader_of( + &fx.cluster.nodes[session_node], + nodedb_cluster::calvin::SEQUENCER_GROUP_ID, + ) == fx.cluster.nodes[victim].node_id; + let session = strict_session(&fx.cluster.nodes[session_node]).await; + let txn = format!( + "BEGIN; \ + INSERT INTO {col_a} (id, v) VALUES ('k1', 'a'); \ + INSERT INTO {col_b} (id, v) VALUES ('k2', 'b'); \ + COMMIT" + ); + + let gate_dir = tempfile::tempdir().expect("gate directory"); + let release = gate_dir.path().join("release"); + let parked = gate_dir.path().join("release.parked"); + let committed = { + let _gate = FailGuard::install( + &format!("calvin::before_redo_propose::{col_a}"), + FailAction::WaitForFile(release.clone()), + ); + let commit = session.simple_query(&txn); + let change_leader = async { + wait_for( + "the leader holds the slice's redo after its vote", + HOLD_DEADLINE, + Duration::from_millis(20), + || parked.exists(), + ) + .await; + kill_node(&mut fx.cluster, victim).await; + std::fs::write(&release, b"release").expect("release the held redo"); + }; + let (committed, ()) = tokio::join!(commit, change_leader); + committed + }; + + if victim_leads_sequencer { + let error = committed.expect_err("the submit's answer died with the sequencer leader"); + let code = error.code().map(|code| code.code().to_owned()); + assert_eq!( + code.as_deref(), + Some("57014"), + "a lost submit answer reports an unknown outcome, got: {error:?}" + ); + } else { + committed.expect("the transaction commits under the new leader"); + } + fx.converge().await; + for coll in [&col_a, &col_b] { + fx.wait_rows_on_every_node(&format!("SELECT id FROM {coll}"), 1) + .await; + } + + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs index f6240a566..925c17c4d 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs @@ -14,6 +14,7 @@ mod bitemporal_array_cluster; mod calvin_cdc_net_kinds; mod calvin_cluster_pgwire_e2e; mod calvin_joiner_after_sequencer_compaction; +mod calvin_leader_change_mid_txn; mod calvin_multi_shard_bitemporal_best_effort_restart; mod calvin_multi_shard_bitemporal_restart; mod calvin_multi_shard_redo_restart; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_two_phase.rs b/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_two_phase.rs index 8c7653806..2a1e3c145 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_two_phase.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_two_phase.rs @@ -1,15 +1,15 @@ // SPDX-License-Identifier: BUSL-1.1 -//! A cross-shard, Calvin-committed write goes through the stage/flush seam: -//! the scheduler STAGES the transaction (validate + buffer, no base mutation), -//! then — because the local commit vote is valid — FLUSHES the staged buffer to -//! base. The write is durable and visible only after the flush. +//! A cross-shard, Calvin-committed write goes through the stage/install seam: +//! the scheduler stages the transaction (validate + buffer, no base mutation). +//! On a COMMIT verdict the leader proposes the slice's stamped redo to the data +//! group, and its apply installs the staged writes. The write is visible only +//! after the install. //! -//! The staged buffer + verdict-driven flush/drop live on the `!Send` Data-Plane -//! core, so this asserts the flush FIRED via the node-global -//! `calvin.counters.commits_flushed` counter (incremented once per staged apply the -//! per-vShard scheduler resolved to commit) plus the functional proof that the -//! flushed write is visible. +//! The staged buffer lives on the `!Send` Data-Plane core, so this asserts the +//! commit fired via the node-global `calvin.counters.commits_flushed` counter +//! (incremented once per staged txn the scheduler resolved to commit) plus the +//! functional proof that the installed write is visible. use crate::common; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_write_versions.rs b/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_write_versions.rs index b4ee53f00..0bbdf751d 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_write_versions.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_write_versions.rs @@ -5,13 +5,10 @@ //! no version at all (fast-path writes and reads already do). //! //! The version index itself lives on the `!Send` Data-Plane core and its -//! readers are test-only, so this asserts the recording FIRED via the -//! node-global `calvin.counters.write_versions_recorded` counter, which the per-vShard -//! scheduler increments once per committed Calvin apply for which it dispatched -//! a write-version record op (at the CalvinApplied WAL LSN). The counter is the -//! standalone-observable proof that a cross-shard-committed write now advances -//! the version index; the end-to-end serializability regression (via read-set -//! validation) is covered separately once validation is enforcing. +//! readers are test-only, so this asserts the recording fired via the +//! node-global `calvin.counters.write_versions_recorded` counter. The data +//! group's apply loop increments it once per committed slice whose stamped +//! redo installed, and the install records the slice's write versions. use crate::common; diff --git a/nodedb-cluster/src/multi_raft/peer_contact.rs b/nodedb-cluster/src/multi_raft/peer_contact.rs index d858ebca4..6adfbae23 100644 --- a/nodedb-cluster/src/multi_raft/peer_contact.rs +++ b/nodedb-cluster/src/multi_raft/peer_contact.rs @@ -27,6 +27,15 @@ impl MultiRaft { .map(|node| node.current_term()) } + /// Index of the no-op this node appended when it won its current term in + /// `group_id`, while it leads the group, else `None`. Every entry of an + /// earlier term in its log sits below it and commits with it. + pub fn term_start_index(&self, group_id: u64) -> Option { + self.groups + .get(&group_id) + .and_then(|node| node.term_start_index()) + } + /// Whether this node heard from the leader of `group_id` within /// `window`. The leader itself always has. Unlike a staleness bound, a /// replica that is merely behind on apply still counts as in contact. diff --git a/nodedb-cluster/src/multi_raft/proposals.rs b/nodedb-cluster/src/multi_raft/proposals.rs index 8b55bca80..5d8033ee7 100644 --- a/nodedb-cluster/src/multi_raft/proposals.rs +++ b/nodedb-cluster/src/multi_raft/proposals.rs @@ -30,10 +30,8 @@ impl MultiRaft { /// Maps the vshard to its Raft group via the routing table and reuses the /// existing local leader-role check — no new election. Returns `false` when /// the vshard has no group mapping or this node is a follower/learner for - /// the owning group. Used by the Calvin scheduler to stamp the per-node, - /// non-replicated `is_group_leader` dispatch flag so the OLLP optimistic-lock - /// verification runs only on the leader while every replica applies the same - /// predicted write-set (determinism). + /// the owning group. The Calvin scheduler proposes its owed sequencer + /// entries only while this holds. pub fn vshard_role_is_leader(&self, vshard_id: u32) -> bool { match self .routing diff --git a/nodedb-cluster/src/rpc_codec/data_plane_error.rs b/nodedb-cluster/src/rpc_codec/data_plane_error.rs index f67852305..e1157d806 100644 --- a/nodedb-cluster/src/rpc_codec/data_plane_error.rs +++ b/nodedb-cluster/src/rpc_codec/data_plane_error.rs @@ -237,6 +237,12 @@ pub enum DataPlaneErrorCode { object: String, detail: String, }, + /// The serving core fail-stopped and applied nothing. Another replica, or + /// the same one after a restart, serves the request. + CoreFailStopped { + core_id: u64, + detail: String, + }, } /// Wire mirror of `nodedb_types::text_search::TextColumnFault`. diff --git a/nodedb-cluster/src/transport/client/attempt.rs b/nodedb-cluster/src/transport/client/attempt.rs index 800562a8d..3ad90fd5c 100644 --- a/nodedb-cluster/src/transport/client/attempt.rs +++ b/nodedb-cluster/src/transport/client/attempt.rs @@ -225,7 +225,8 @@ impl NexarTransport { /// The outer error is a link failure. The inner result is the peer's /// reply. A stream the peer finished without a reply is an inner error: /// the peer refused the request before its handler, and a resend gets - /// the same answer. + /// the same answer. The peer resets a stream whose handler started and + /// sent no answer, and that reset is a link failure of a written request. async fn try_send_once( &self, target: u64, diff --git a/nodedb-cluster/src/transport/client/transport.rs b/nodedb-cluster/src/transport/client/transport.rs index 742e31357..84cdbe740 100644 --- a/nodedb-cluster/src/transport/client/transport.rs +++ b/nodedb-cluster/src/transport/client/transport.rs @@ -442,7 +442,8 @@ pub(super) mod tests { /// The group `EchoHandler` does not host. const UNHOSTED_GROUP: u64 = 404; - /// The group whose vote requests `EchoHandler` answers after [`STALL`]. + /// The group whose vote and snapshot requests `EchoHandler` answers + /// after [`STALL`]. pub(crate) const STALLED_GROUP: u64 = 503; /// How long `EchoHandler` holds a vote request of [`STALLED_GROUP`]. @@ -486,6 +487,12 @@ pub(super) mod tests { vote_granted: true, })) } + RaftRpc::InstallSnapshotRequest(req) if req.group_id == STALLED_GROUP => { + tokio::time::sleep(STALL).await; + Ok(RaftRpc::InstallSnapshotResponse(InstallSnapshotResponse { + term: req.term, + })) + } RaftRpc::InstallSnapshotRequest(req) => { Ok(RaftRpc::InstallSnapshotResponse(InstallSnapshotResponse { term: req.term, @@ -1034,4 +1041,36 @@ pub(super) mod tests { assert_eq!(breaker.state(1), CircuitState::Closed); assert_eq!(breaker.failure_count(1), 0); } + + /// A node that shuts down while its handler runs a request leaves the + /// request unanswered. The handler can have run it, so the sender never + /// reads it as a refusal and never resends it. + #[tokio::test] + async fn a_request_whose_node_shuts_down_mid_handler_ends_unanswered() { + let (_server, client, shutdown) = serve_echo().await; + let req = RaftRpc::InstallSnapshotRequest(InstallSnapshotRequest { + term: 7, + leader_id: 2, + last_included_index: 500, + last_included_term: 6, + offset: 0, + data: Vec::new(), + done: true, + group_id: STALLED_GROUP, + total_size: 0, + voters: Vec::new(), + learners: Vec::new(), + }); + assert!(!req.resend_safe()); + let send = tokio::spawn(async move { client.send_rpc(1, req).await }); + tokio::time::sleep(STALL / 4).await; + shutdown.send(true).expect("signal shutdown"); + + let err = send.await.expect("send task").unwrap_err(); + + assert!( + matches!(err, ClusterError::Unanswered { node_id: 1, .. }), + "{err:?}" + ); + } } diff --git a/nodedb-cluster/src/transport/mod.rs b/nodedb-cluster/src/transport/mod.rs index 65a244415..5e0cecad5 100644 --- a/nodedb-cluster/src/transport/mod.rs +++ b/nodedb-cluster/src/transport/mod.rs @@ -9,6 +9,7 @@ mod identity_admission; pub mod peer_identity_store; pub mod peer_identity_verifier; pub mod pinned_verifier; +mod reply_stream; pub mod rpc_handler; pub mod server; mod shuffle_drain; diff --git a/nodedb-cluster/src/transport/reply_stream.rs b/nodedb-cluster/src/transport/reply_stream.rs new file mode 100644 index 000000000..6d9cfe8c3 --- /dev/null +++ b/nodedb-cluster/src/transport/reply_stream.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The send half of an inbound RPC stream, reset when its request went +//! unanswered. +//! +//! A sender reads a stream finished without a reply as a refusal before the +//! handler: the request never ran, and a resend is safe. quinn finishes a +//! dropped `SendStream`, so a stream dropped once its handler started would +//! read the same way. A node that shuts down mid-request, a reply that fails +//! to write, and a handler panic all drop it there. +//! +//! [`ReplyStream`] closes that gap. Once armed, it resets the stream on drop +//! unless the request was answered. The sender reads a reset as a written +//! request with no answer, an unknown outcome it never resends. + +use std::ops::{Deref, DerefMut}; + +/// The QUIC stream error code of a request whose handler started and that +/// got no answer. +const UNANSWERED_RESET_CODE: u32 = 1; + +/// The send half of one inbound RPC stream. +pub(super) struct ReplyStream { + send: quinn::SendStream, + /// Whether the handler started and no answer went out yet. + armed: bool, +} + +impl ReplyStream { + pub(super) fn new(send: quinn::SendStream) -> Self { + Self { send, armed: false } + } + + /// Mark the request as handed to its handler. A drop after this point + /// resets the stream. + pub(super) fn arm(&mut self) { + self.armed = true; + } + + /// Mark the request as answered. The drop then leaves the stream to + /// quinn's finish. + pub(super) fn answered(&mut self) { + self.armed = false; + } +} + +impl Deref for ReplyStream { + type Target = quinn::SendStream; + + fn deref(&self) -> &Self::Target { + &self.send + } +} + +impl DerefMut for ReplyStream { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.send + } +} + +impl Drop for ReplyStream { + fn drop(&mut self) { + if self.armed { + // A stream already closed by the connection needs no reset. + let _ = self + .send + .reset(quinn::VarInt::from_u32(UNANSWERED_RESET_CODE)); + } + } +} diff --git a/nodedb-cluster/src/transport/server.rs b/nodedb-cluster/src/transport/server.rs index ec2cd16d6..f694fb2d1 100644 --- a/nodedb-cluster/src/transport/server.rs +++ b/nodedb-cluster/src/transport/server.rs @@ -50,6 +50,7 @@ use crate::transport::rpc_handler::RaftRpcHandler; use crate::wire_version::handshake_io::{local_version_range, perform_version_handshake_server}; use super::frame_io::{finish_stream, read_envelope, reply_and_finish, write_rpc_frame}; +use super::reply_stream::ReplyStream; use super::shuffle_drain::drain_shuffle_push; use super::stream_dispatch; use super::stream_identity::{reject_peer_identity, verify_stream_identity}; @@ -220,9 +221,10 @@ struct StreamContext { /// module docstring for the rationale. async fn handle_stream( ctx: StreamContext, - mut send: quinn::SendStream, + send: quinn::SendStream, mut recv: quinn::RecvStream, ) -> Result<()> { + let mut send = ReplyStream::new(send); let StreamContext { handler, auth, @@ -279,6 +281,9 @@ async fn handle_stream( fields.from_node_id, &request, )?; + // The handler can run the request from here on. A stream that ends + // without an answer is reset, so the sender never resends it. + send.arm(); // 4b. Streaming path: an `ExecuteStreamRequest` produces a multi-frame // response — N `RPC_EXECUTE_STREAM_CHUNK` envelopes (each written @@ -338,11 +343,15 @@ async fn handle_stream( reply_and_finish(&mut send, &auth, &response, "response").await }; - tokio::select! { + let outcome = tokio::select! { biased; - _ = shutdown.changed() => Ok(()), - result = work => result, + _ = shutdown.changed() => Ok(false), + result = work => result.map(|()| true), + }; + if let Ok(true) = outcome { + send.answered(); } + outcome.map(|_| ()) } /// The frame that answers a one-shot request: the handler's response, or a diff --git a/nodedb-physical/src/physical_plan/calvin_reply.rs b/nodedb-physical/src/physical_plan/calvin_reply.rs index 7e61ac143..335f0baf4 100644 --- a/nodedb-physical/src/physical_plan/calvin_reply.rs +++ b/nodedb-physical/src/physical_plan/calvin_reply.rs @@ -125,6 +125,9 @@ pub struct CalvinInstall { pub epoch_system_ms: i64, /// The reply the install answers with. pub reply: CalvinReplySpec, + /// Whether the install raises the tenant's write mark. A slice that only + /// installs schema raises none. + pub user_write: bool, } /// The answer of `MetaOp::CalvinResolve`: the slice's resolved redo record diff --git a/nodedb-physical/src/physical_plan/meta.rs b/nodedb-physical/src/physical_plan/meta.rs index 54fa96f3e..0190fab5f 100644 --- a/nodedb-physical/src/physical_plan/meta.rs +++ b/nodedb-physical/src/physical_plan/meta.rs @@ -314,10 +314,12 @@ pub enum MetaOp { /// /// The Calvin scheduler dispatches this variant after lock acquisition for /// transactions whose read/write set is fully known at submission time (the - /// common case). The Data Plane handler validates the read-set and stages - /// `plans` without mutating base. Once the global verdict is commit, the - /// scheduler resolves the staged plans into a redo record, appends it, and - /// flushes it (`CalvinResolve`, `CalvinFlush`). + /// common case). Only the vShard's data-group leader dispatches it. The + /// Data Plane handler validates the read-set and stages `plans` without + /// mutating base. Once the global verdict is commit, the scheduler + /// resolves the staged plans into a redo record (`CalvinResolve`) and + /// proposes it to the data group. Every replica installs it from the log + /// (`ApplyTransactionRedo`). /// /// NOTE: This variant occupies the same msgpack positional tag as the /// original `CalvinExecute` variant it replaces, preserving wire @@ -337,13 +339,6 @@ pub enum MetaOp { /// the wall clock independently. Wire-additive: defaults to 0 on decode /// of older entries. epoch_system_ms: i64, - /// Whether THIS node is the leader of the data-group owning this vshard, - /// stamped by the dispatching scheduler. OLLP verification (`actual != - /// predicted` → `OllpRetryRequired`) is leader-only; followers apply the - /// carried `ollp_predicted_surrogates` verbatim so every replica mutates - /// the identical set (Calvin determinism). Per-node, never replicated. - /// Wire-additive: defaults to `false` on decode of older entries. - is_group_leader: bool, /// The transaction's LSN-versioned read-set from the replicated `TxClass`. /// Each participant checks its own vShard's reads at apply and records /// whether they were still current, without gating apply. Wire-additive: @@ -405,13 +400,6 @@ pub enum MetaOp { /// Wall-clock ms read once on the sequencer leader at epoch creation. /// Same semantics as `CalvinExecuteStatic::epoch_system_ms`. epoch_system_ms: i64, - /// Whether THIS node is the leader of the data-group owning this - /// vshard. Same semantics as `CalvinExecuteStatic::is_group_leader`: - /// gates the LEADER-ONLY OLLP verification + `OllpRetryRequired`; every - /// replica applies the carried `ollp_predicted_surrogates` set verbatim - /// for determinism. Per-node, non-replicated; wire-additive (defaults to - /// `false`). - is_group_leader: bool, }, /// Rebuild a collection's indexes (HNSW, full-text, graph CSR) on @@ -505,22 +493,6 @@ pub enum MetaOp { savepoint: u64, }, - /// Record the per-key write versions of a committed Calvin transaction's - /// locally-applied write plans. - /// - /// The scheduler stamps the transaction's committed LSN onto this op's - /// `wal_lsn` and dispatches it back to the same core once the flush - /// completes. The core funnels `plans` through the shared write-version - /// recorder at that LSN — the same shard-local WAL-LSN space the - /// single-shard fast path and read watermarks use. Records only: no base - /// mutation, no WAL append, no event emission. - RecordCalvinWriteVersions { - /// Tenant scope for all plans. - tenant_id: TenantId, - /// The locally-applied write plans whose keys' versions are recorded. - plans: Vec, - }, - /// Install a committed Calvin transaction's redo record on base storage. /// /// `CalvinExecuteStatic` STAGES the transaction's plans without mutating @@ -544,10 +516,11 @@ pub enum MetaOp { /// Discard the staged writes of a Calvin transaction. /// - /// Dispatched by the scheduler when the local commit vote resolves to abort. - /// The handler removes the staged plans keyed by `(epoch, position)` and - /// fires nothing — no base mutation, no side effects. Absent key is an - /// idempotent no-op. + /// Dispatched by the leader's scheduler when the slice ends with no log + /// entry (an abort verdict, or a COMMIT for a slice with no write), and + /// by a demoted leader for state it staged. The handler removes the + /// staged plans keyed by `(epoch, position)` and fires nothing — no base + /// mutation, no side effects. Absent key is an idempotent no-op. CalvinDrop { epoch: u64, position: u32 }, /// Resolve a committing transaction's staged writes into ONE replayable diff --git a/nodedb-raft/src/node/core.rs b/nodedb-raft/src/node/core.rs index 6485f7c30..d895ec21a 100644 --- a/nodedb-raft/src/node/core.rs +++ b/nodedb-raft/src/node/core.rs @@ -119,6 +119,9 @@ pub struct RaftNode { /// When this node won its current leadership term. `None` off the /// leader path. A voter not yet heard this term counts as last seen here. pub(super) leader_since: Option, + /// Index of the no-op this node appended when it won its current + /// leadership term. `None` off the leader path. + pub(super) term_start_index: Option, /// Leader-lease rounds and anchor (see [`super::leader_lease`]). pub(super) lease: LeaseState, /// Until this instant the node refuses every vote but a transfer vote. @@ -182,6 +185,7 @@ impl RaftNode { last_quorum_contact: None, quorum_window: Vec::new(), leader_since: None, + term_start_index: None, lease: LeaseState::new(), boot_vote_fence: now + config.election_timeout_max, compaction_ceiling: None, @@ -254,6 +258,16 @@ impl RaftNode { self.log.last_index() } + /// Index of the no-op this leader appended when it won its current + /// term. Every entry of an earlier term in its log sits below it and + /// commits with it. `None` when this node does not lead. + pub fn term_start_index(&self) -> Option { + if self.role != NodeRole::Leader { + return None; + } + self.term_start_index + } + /// Override election deadline (for testing). pub fn election_deadline_override(&mut self, deadline: Instant) { self.election_deadline = deadline; diff --git a/nodedb-raft/src/node/internal.rs b/nodedb-raft/src/node/internal.rs index defb9cb73..dc1cf1fa1 100644 --- a/nodedb-raft/src/node/internal.rs +++ b/nodedb-raft/src/node/internal.rs @@ -148,6 +148,7 @@ impl RaftNode { self.last_quorum_contact = None; self.quorum_window.clear(); self.leader_since = None; + self.term_start_index = None; self.persist_hard_state(); self.reset_election_timeout(); @@ -198,7 +199,9 @@ impl RaftNode { index: self.log.last_index() + 1, data: Vec::new(), }; + let noop_index = noop.index; let _ = self.log.append(noop); + self.term_start_index = Some(noop_index); // Single-voter cluster: the no-op commits once it is durable here. if self.config.cluster_size() == 1 { @@ -525,6 +528,25 @@ mod tests { ); } + /// A leader names the index of the no-op it appended when it won its + /// term. Later proposals leave it unchanged, and a step-down clears it. + #[test] + fn a_leader_names_its_term_start_index() { + let mut node = RaftNode::new(test_config(1, vec![]), MemStorage::new()); + assert_eq!(node.term_start_index(), None, "a follower names none"); + force_election(&mut node); + let noop = node.last_log_index(); + assert_eq!(node.term_start_index(), Some(noop)); + + node.propose(b"write".to_vec()) + .expect("single voter commits immediately"); + assert_eq!(node.term_start_index(), Some(noop)); + + let term = node.current_term(); + node.become_follower(term + 1); + assert_eq!(node.term_start_index(), None); + } + /// A committed range below the log's first available index is a hard /// error in `Ready`, never a silently empty delivery. #[test] diff --git a/nodedb/src/bootstrap/calvin_catch_up.rs b/nodedb/src/bootstrap/calvin_catch_up.rs index 2ebe00f5f..26b853a13 100644 --- a/nodedb/src/bootstrap/calvin_catch_up.rs +++ b/nodedb/src/bootstrap/calvin_catch_up.rs @@ -118,9 +118,9 @@ mod tests { vshard_id, epoch: 9, position: 1, - reason: "flush_failed", - step: "flush", - error: "flush returned Error".to_string(), + reason: "redo_apply_failed", + step: "redo_apply", + error: "redo install did not become durable".to_string(), } } diff --git a/nodedb/src/bridge/envelope/error_code.rs b/nodedb/src/bridge/envelope/error_code.rs index a63576fcb..d21d282b6 100644 --- a/nodedb/src/bridge/envelope/error_code.rs +++ b/nodedb/src/bridge/envelope/error_code.rs @@ -25,6 +25,10 @@ pub enum ErrorCode { /// a retry channel can tell a retry apart from a permanent refusal instead /// of collapsing both into a terminal rejection. RetryableRefusal { reason: String }, + /// The core fail-stopped: its state is unknown until a restart rebuilds + /// it from the WAL. It refused the request and applied nothing. Another + /// replica, or this one after its restart, serves the request. + CoreFailStopped { core_id: usize, detail: String }, /// A sync frame the validator refused for good. Nothing applied. The /// stream's high-water mark advanced to `provenance`, so the frame is /// never admitted again, and `applied_seq` is the mark after the refusal. diff --git a/nodedb/src/control/change_stream/mod.rs b/nodedb/src/control/change_stream/mod.rs index 667101373..9bc5c58f8 100644 --- a/nodedb/src/control/change_stream/mod.rs +++ b/nodedb/src/control/change_stream/mod.rs @@ -6,9 +6,9 @@ pub mod stream; pub use journal::ChangeJournal; pub use live_set::LiveSubscriptionSet; +pub(crate) use stream::PositionedChange; pub use stream::{ ChangeCursor, ChangeEvent, ChangeOperation, ChangePartition, ChangeStream, ChangeStreamError, CursorParseError, CursorStep, ReplayError, ReplaySnapshot, ReplayStart, ReplayedChange, SequencedChangeEvent, Subscription, }; -pub(crate) use stream::{ChangeRun, PositionedChange}; diff --git a/nodedb/src/control/change_stream/stream/mod.rs b/nodedb/src/control/change_stream/stream/mod.rs index 755d42a82..fdf073252 100644 --- a/nodedb/src/control/change_stream/stream/mod.rs +++ b/nodedb/src/control/change_stream/stream/mod.rs @@ -12,7 +12,7 @@ pub mod types; pub use bus::ChangeStream; pub use cursor::{ChangeCursor, ChangePartition, CursorParseError, CursorStep}; pub use error::ChangeStreamError; -pub(crate) use ring::{ChangeRun, PositionedChange}; +pub(crate) use ring::PositionedChange; pub use ring::{ReplayError, ReplaySnapshot, ReplayStart, ReplayedChange}; pub use subscription::Subscription; pub use types::{ChangeEvent, ChangeOperation, SequencedChangeEvent}; diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/apply_result.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/apply_result.rs new file mode 100644 index 000000000..464500e25 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/apply_result.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! What a committed slice's install answers to its coordinator: the local +//! applied-result sidecar entry, and the result its `CompletionAck` carries. + +use crate::bridge::envelope::{Response, Status}; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; +use crate::control::state::{AckSlice, CalvinAckResult, CalvinApplyResult}; + +use super::scheduler::Scheduler; + +impl Scheduler { + /// Deposit a primary slice's applied `reply` into this node's sidecar, + /// before its `CompletionAck` fires the coordinator's waiter. + /// + /// A multi-collection cross-shard COMMIT has many primary-write + /// participants, each a plain affected-count write, and they coalesce: + /// the first answer stands, with every participant's timeseries install + /// counts merged in. Only two participants that each carry RETURNING + /// rows conflict: a cross-shard RETURNING union is unsupported, so the + /// statement fails loudly rather than return one shard's rows. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn deposit_reply( + &self, + txn_id: TxnId, + reply: &Response, + returning: bool, + ) { + use std::collections::hash_map::Entry; + + let response = statement_reply(reply.clone()); + let key = nodedb_cluster::calvin::TxnId::new(txn_id.epoch, txn_id.position); + let vshard_id = self.vshard_id; + self.shared + .calvin + .apply_results + .deposit_with(key, |entry| match entry { + Entry::Vacant(slot) => { + slot.insert(CalvinApplyResult::Single { + response, + has_returning: returning, + }); + } + Entry::Occupied(mut slot) => { + let existing_returning = matches!( + slot.get(), + CalvinApplyResult::Single { + has_returning: true, + .. + } + ); + let already_conflict = matches!(slot.get(), CalvinApplyResult::Conflict); + if already_conflict { + // A RETURNING union was already recorded; it stays one. + } else if returning && existing_returning { + tracing::error!( + epoch = txn_id.epoch, + position = txn_id.position, + vshard = vshard_id, + "two RETURNING-bearing participants for one Calvin txn — cross-shard \ + RETURNING union unsupported" + ); + slot.insert(CalvinApplyResult::Conflict); + } else if returning { + // The incoming participant carries the rows; the held + // entry was a plain affected-count sibling. Rows win. + slot.insert(CalvinApplyResult::Single { + response, + has_returning: true, + }); + } else { + merge_install_counts(slot.get_mut(), &response); + } + } + }); + } + + /// The result a slice's `CompletionAck` carries for its applied `reply`: + /// the install's timeseries counts, and the rows of a RETURNING slice. + /// The rows are bounded by the query result limit and by the ack's + /// frame budget, whichever is lower. Every replica renders the same + /// reply, so every replica owes the same bytes. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn ack_result_of( + &self, + reply: &Response, + slice: AckSlice, + ) -> Vec { + let limit = self + .shared + .tuning + .network + .max_query_result_bytes + .min(crate::control::state::ACK_ROWS_FRAME_BUDGET); + CalvinAckResult::of(reply, slice, limit) + .to_bytes() + .unwrap_or_else(|error| { + tracing::warn!( + vshard_id = self.vshard_id, + %error, + "calvin: the completion ack carries no report" + ); + Vec::new() + }) + } +} + +/// Fold the timeseries install counts `incoming` answers into the plain +/// result `held`. A RETURNING result keeps its rows untouched. +fn merge_install_counts(held: &mut CalvinApplyResult, incoming: &Response) { + let CalvinApplyResult::Single { + response, + has_returning: false, + } = held + else { + return; + }; + if let Some(merged) = crate::engine::timeseries::install_counts::merge_count_payloads( + response.payload.as_bytes(), + incoming.payload.as_bytes(), + ) { + response.payload = crate::bridge::envelope::Payload::from_vec(merged); + } +} + +/// The response the statement drains. An install whose reply failed to +/// render answers `Ok` with the render error in `error_code`. The statement +/// reports that error. +fn statement_reply(mut response: Response) -> Response { + if response.status == Status::Ok && response.error_code.is_some() { + response.status = Status::Error; + response.payload = crate::bridge::envelope::Payload::empty(); + } + response +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bridge::envelope::ErrorCode; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + error_response, staged_response, + }; + + fn installed_with_counts(collection: &str, accepted: u64, rejected: u64) -> Response { + use crate::engine::timeseries::install_counts::{TsInstallCount, TsInstallCounts}; + let mut response = staged_response(Status::Ok, None); + response.payload = crate::bridge::envelope::Payload::from_vec( + TsInstallCounts::new(vec![TsInstallCount { + collection: collection.into(), + accepted, + rejected, + }]) + .to_bytes() + .expect("encode install counts"), + ); + response + } + + /// Two plain participants of one Calvin transaction each installed a + /// timeseries batch. The coordinator's result carries both installs' + /// apply counts, so COMMIT reports every participant's rejected rows. A + /// RETURNING result keeps its rows. + #[test] + fn plain_participants_merge_their_install_counts() { + use crate::engine::timeseries::install_counts::TsInstallCounts; + let mut held = CalvinApplyResult::Single { + response: installed_with_counts("cpu", 2, 1), + has_returning: false, + }; + merge_install_counts(&mut held, &installed_with_counts("mem", 3, 2)); + let CalvinApplyResult::Single { response, .. } = &held else { + panic!("a merged result stays single"); + }; + let totals = TsInstallCounts::from_payload(response.payload.as_bytes()) + .expect("install counts") + .by_collection(); + assert_eq!(totals.get("cpu"), Some(&(2, 1))); + assert_eq!(totals.get("mem"), Some(&(3, 2))); + + let rows = crate::bridge::envelope::Payload::from_vec(vec![0x90]); + let mut returning = CalvinApplyResult::Single { + response: Response { + payload: rows, + ..installed_with_counts("cpu", 1, 0) + }, + has_returning: true, + }; + merge_install_counts(&mut returning, &installed_with_counts("mem", 1, 1)); + let CalvinApplyResult::Single { response, .. } = &returning else { + panic!("a RETURNING result stays single"); + }; + assert_eq!(response.payload.as_bytes(), &[0x90]); + } + + /// A render error on an installed slice reaches the statement as a + /// typed error. + #[test] + fn a_render_error_on_an_installed_slice_becomes_the_statement_error() { + let mut response = error_response(ErrorCode::Internal { + detail: "render".to_string(), + }); + response.status = Status::Ok; + + let reply = statement_reply(response); + + assert_eq!(reply.status, Status::Error); + assert!(matches!( + reply.error_code.as_deref(), + Some(ErrorCode::Internal { .. }) + )); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/catch_up.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/catch_up.rs index d02acd9a6..ce1ca86d1 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/catch_up.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/catch_up.rs @@ -253,6 +253,7 @@ mod tests { use nodedb_types::TenantId; use nodedb_types::id::DatabaseId; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::lead_data_group; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ build_test_scheduler, build_test_scheduler_with_data_side, fill_tenant_inflight, make_sequenced_txn, make_validate_only_txn, test_coll_vshard, @@ -578,6 +579,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); arm_two_dropped_stage_batches(&scheduler); let shared = std::sync::Arc::clone(&scheduler.shared); fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); @@ -602,6 +604,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); let (_idx0, idx1) = arm_two_dropped_stage_batches(&scheduler); let shared = std::sync::Arc::clone(&scheduler.shared); fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_redo.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_redo.rs index 555471d6c..4cea57a79 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_redo.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_redo.rs @@ -1,39 +1,41 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Redo-record resolution for a committed staged static Calvin transaction. +//! Redo resolution for a committed Calvin slice on the data-group leader. //! -//! A committed static Calvin dispatch stages its transaction on the Data -//! Plane (validate the read-set + buffer the plans, no base mutation). Once -//! the local commit vote is known (`resolve_staged_commit`), this module -//! drives the resolve step: dispatch `MetaOp::CalvinResolve` to reconstitute -//! the staged post-images as one replayable `RedoRecord`, WAL-append that -//! record (restoring restart durability for this vShard's slice of the -//! commit), then queue the flush for its sequencer-order turn -//! ([`super::flush_turn`]); `finish_resolved_commit` / `commit_apply_tail` -//! complete it. +//! Once the global verdict is COMMIT, the leader dispatches +//! `MetaOp::CalvinResolve` to reconstitute the staged post-images as one +//! replayable `RedoRecord`. It stamps the record with the slice's sequencer +//! position, attaches the transaction's messages and dedup key on the +//! participant that carries them, and proposes the record to the vShard's +//! data group (see [`super::redo_propose`]). Every replica, this one +//! included, installs it from the log. +use nodedb_physical::physical_plan::PhysicalPlan; +use nodedb_physical::physical_plan::meta::MetaOp; + +use super::super::types::CommitState; use super::deferred::{DispatchOutcome, DispatchStep}; use super::halt::{HaltReason, HaltStep, error_response_text}; +use super::redo_propose::OwedRedo; use super::scheduler::Scheduler; use crate::bridge::envelope::{Response, Status}; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; -use crate::control::server::dispatch_utils::MintedRecords; +use crate::control::wal_replication::CollectionIncarnation; +use crate::control::wal_replication::encode::{CalvinEntryStamps, RedoEntryTarget}; +use crate::control::wal_replication::transaction_redo::{CalvinSlice, TransactionRedoPayload}; +use crate::control::wal_replication::types::CalvinRedoMeta; use crate::types::VShardId; use crate::wal::{CalvinStamp, RedoRecord}; -use nodedb_physical::physical_plan::PhysicalPlan; -use nodedb_physical::physical_plan::meta::MetaOp; impl Scheduler { /// Handle the `MetaOp::CalvinResolve` response: decode the resolved - /// `RedoRecord`, attach the transaction's messages on the participant that - /// carries them, WAL-append it (unless it holds no op and no message), - /// then dispatch the flush that installs it, stamped with that record's - /// LSN. + /// `RedoRecord`, stamp it, attach the transaction's messages and dedup + /// key on the participant that carries them, and propose it. /// - /// The verdict is already COMMIT, so a skipped resolve will tear the - /// committed txn on this replica. A non-`Ok` response, a decode failure, - /// or a WAL-append failure halts the scheduler: the txn keeps its - /// `pending` entry and locks, and its position stays unapplied. + /// The verdict is already COMMIT, so a skipped resolve tears the + /// committed txn. A non-`Ok` response or a record that cannot be read + /// halts the scheduler: the txn keeps its `pending` entry and locks, and + /// its position stays unapplied. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn finish_redo_resolve( &mut self, txn_id: TxnId, @@ -48,226 +50,134 @@ impl Scheduler { ); return; } + match self.owed_redo(txn_id, &response) { + Ok(owed) => { + if let Some(pending) = self.pending.get_mut(&txn_id) { + pending.redo = Some(owed); + pending.commit_state = CommitState::AwaitingRedoApply { proposed: None }; + } + self.propose_calvin_redo(txn_id); + } + Err(error) => self.halt_apply( + txn_id, + HaltReason::ResolveFailed, + HaltStep::Resolve, + error.to_string(), + ), + } + } - // The answer carries the slice's staged reply beside the record. The - // flush renders the reply from the core's staged entry. + /// The stamped redo the committed slice `txn_id` proposes, from its + /// resolve answer `response`. + fn owed_redo(&self, txn_id: TxnId, response: &Response) -> crate::Result { + // The answer carries the slice's staged reply beside the record. Every + // replica renders the reply after its install. let resolved = zerompk::from_msgpack::( response.payload.as_bytes(), - ); - let decoded = match resolved { - Ok(resolved) => RedoRecord::from_bytes(&resolved.redo), - Err(e) => Err(crate::Error::Serialization { - format: "msgpack".into(), - detail: format!("CalvinResolve answer: {e}"), - }), - }; - let mut redo = match decoded { - Ok(r) => r, - Err(e) => { - self.halt_apply( - txn_id, - HaltReason::ResolveFailed, - HaltStep::Resolve, - format!("CalvinResolve redo record decode failed: {e}"), - ); - return; - } - }; - let Some(pending) = self.pending.get(&txn_id) else { - // Txn state was reclaimed out from under us (must not happen — - // locks are held until `on_txn_complete`); complete defensively. - self.metrics.record_completed(); - self.on_txn_complete(txn_id); - return; - }; - let tenant_id = pending.txn.tx_class.tenant_id; - let database_id = pending.txn.tx_class.database_id; - let event_source = - super::request::slice_event_source(&pending.txn.tx_class, &pending.flush_scope); - let commit_hlc = self - .cut_floors - .commit_hlc(pending.txn.epoch, pending.txn.epoch_system_ms); - // The stamp carries what the slice folds, so the live install and - // restart replay fold at this record's LSN. + ) + .map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("CalvinResolve answer: {e}"), + })?; + let mut redo = RedoRecord::from_bytes(&resolved.redo)?; + let pending = self + .pending + .get(&txn_id) + .ok_or_else(|| missing_pending_error(txn_id))?; + let tx_class = &pending.txn.tx_class; + // The stamp names the position every replica claims at install. redo.calvin_stamp = Some(CalvinStamp { epoch: txn_id.epoch, position: txn_id.position, vshard_id: self.vshard_id, - collections: pending.flush_scope.collections.clone(), - sum_targets: pending.flush_scope.sum_targets.clone(), }); // The participants whose records carry the messages and the applied - // key. A class whose write vShards cannot be derived halts the resolve. - let tx_class = &pending.txn.tx_class; - let homes = tx_class + // key. A class whose write vShards cannot be derived fails. + let publish_home = tx_class .publish_vshard() - .and_then(|publish| Ok((publish, tx_class.applied_key_home()?))); - let (publish_home, applied_key_home) = match homes { - Ok(homes) => homes, - Err(e) => { - self.halt_apply( - txn_id, - HaltReason::ResolveFailed, - HaltStep::Resolve, - format!("Calvin transaction write vShards underivable: {e}"), - ); - return; - } - }; + .map_err(|e| crate::Error::Internal { + detail: format!("Calvin transaction write vShards underivable: {e}"), + })?; + let applied_key_home = tx_class + .applied_key_home() + .map_err(|e| crate::Error::Internal { + detail: format!("Calvin transaction write vShards underivable: {e}"), + })?; // One participant's record carries the messages the transaction's // trigger bodies published, so they commit once, with its writes. + // Every replica stamps their position from the entry's log place. if publish_home == Some(self.vshard_id) { - match crate::wal::RedoPublish::decode_all(&pending.txn.tx_class.publishes) { - Ok(mut publishes) => { - // Named by the transaction's sequencer position on its - // vShard's Calvin partition, the same on every replica. - crate::wal::RedoPublish::stamp_all( - &mut publishes, - crate::wal::PublishPosition { - partition: crate::event::cdc::position::calvin_partition( - self.vshard_id, - ), - epoch: 0, - index: txn_id.epoch, - base: u64::from(txn_id.position), - }, - ); - redo.publishes = publishes; - } - Err(e) => { - self.halt_apply( - txn_id, - HaltReason::ResolveFailed, - HaltStep::Resolve, - format!("Calvin transaction publishes decode failed: {e}"), - ); - return; - } - } + redo.publishes = crate::wal::RedoPublish::decode_all(&tx_class.publishes)?; } // One participant's record carries the dedup key of the cross-shard // request the transaction applies, so the key is durable exactly when // the request's writes are. if applied_key_home == Some(self.vshard_id) { - match zerompk::from_msgpack::( - &pending.txn.tx_class.applied_key, - ) { - Ok(key) => redo.cross_shard_applied = Some(key), - Err(e) => { - self.halt_apply( - txn_id, - HaltReason::ResolveFailed, - HaltStep::Resolve, - format!("Calvin transaction applied key decode failed: {e}"), - ); - return; - } - } - } - let installs_nothing = - redo.ops.is_empty() && redo.publishes.is_empty() && redo.cross_shard_applied.is_none(); - - // The flush installs these exact bytes, the payload of the record - // appended below. - let redo_bytes = if installs_nothing { - Vec::new() - } else { - match redo.to_bytes() { - Ok(bytes) => bytes, - Err(e) => { - self.halt_apply( - txn_id, - HaltReason::ResolveFailed, - HaltStep::Resolve, - format!("CalvinResolve redo record encode failed: {e}"), - ); - return; - } - } - }; - - // The record's outcome-floor window opens before the append. It stays - // with the pending txn until the flush completes. - let (redo_lsn, redo_records) = if installs_nothing { - (None, None) - } else { - let records = MintedRecords::open(&self.shared.outcome_floor); - // The flush reports no rows beyond the record's own: restart - // replay folds the sum targets from the record's stamp. The record - // is therefore whole at append, and no part follows it. - let appended = records - .appender(&self.shared.wal, crate::wal::manager::NO_APPLY_KEY) - .with_event_source(event_source) - .with_commit_hlc(commit_hlc) - .append_whole_transaction_redo( - tenant_id, - VShardId::new(self.vshard_id), - database_id, - &redo, - ); - match appended { - Ok(lsn) => { - // The txn committed, so its redo record is never - // cancelled. The flush closes it from its outcome. - records.mark_sent(); - // Before the flush, so the position exists before any of - // the txn's change events reach the Event Plane. - self.shared.cdc_router.positions().record_calvin( - lsn.as_u64(), - crate::event::cdc::position::CalvinPosition { - sequencer_epoch: txn_id.epoch, - position: txn_id.position, - }, - ); - self.shared - .cdc_router - .positions() - .record_commit_hlc(lsn.as_u64(), commit_hlc); - (Some(lsn), Some(records)) - } - Err(e) => { - // The txn stays pending and unapplied, and a failed append - // leaves no record for restart replay to reach. - records.settle(); - self.halt_apply( - txn_id, - HaltReason::WalAppendFailed, - HaltStep::RedoAppend, - format!("TransactionRedo WAL append failed: {e}"), - ); - return; - } - } - }; - if let Some(pending) = self.pending.get_mut(&txn_id) { - pending.redo_records = redo_records; - // The commit publishes the rows its redo installs, as a - // data-group `TransactionRedo` apply does. A rolled-back - // transaction never resolves, so it publishes nothing. - pending.change_sets = if redo_bytes.is_empty() { - Vec::new() - } else { - vec![crate::control::server::dispatch_utils::redo_change_set( - &redo_bytes, - )] - }; - pending.flush_scope.redo = redo_bytes; + let key = + zerompk::from_msgpack::(&tx_class.applied_key) + .map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("Calvin transaction applied key: {e}"), + })?; + redo.cross_shard_applied = Some(key); } - - // The flush runs in sequencer order, once every lower txn finished. - self.queue_flush(txn_id, redo_lsn); + let scope = &pending.scope; + let payload = TransactionRedoPayload::from_calvin( + redo, + CalvinSlice { + collections: scope.collections.clone(), + sum_targets: scope.sum_targets.clone(), + identities: scope.identities.clone(), + event_source: super::request::slice_event_source(tx_class, scope), + origin: scope.origin, + meta: CalvinRedoMeta { + epoch_system_ms: pending.txn.epoch_system_ms, + reply: resolved.reply, + primary_write: pending.has_primary_write, + user_write: pending.raises_write_mark, + returning: pending.has_returning, + }, + }, + ); + let incarnations = scope + .collections + .iter() + .map(|collection| CollectionIncarnation { + collection: collection.clone(), + incarnation: tx_class + .incarnations + .iter() + .find(|named| named.collection == *collection) + .map_or(nodedb_types::Hlc::ZERO, |named| named.incarnation), + }) + .collect(); + Ok(OwedRedo { + payload, + stamps: CalvinEntryStamps { + write_hlc: self + .cut_floors + .commit_hlc(pending.txn.epoch, pending.txn.epoch_system_ms), + restore_id: tx_class.restore_id, + metadata_floor: tx_class.metadata_floor, + incarnations, + }, + target: RedoEntryTarget { + tenant_id: tx_class.tenant_id, + database_id: tx_class.database_id, + vshard_id: VShardId::new(self.vshard_id), + }, + attempts: 0, + }) } /// Dispatch `MetaOp::CalvinResolve` to this vShard's core, registering a /// response bridge so the resolve response re-enters the completion loop /// under `CommitState::AwaitingRedoResolve`. /// - /// Mirrors `dispatch_commit_resolution`'s exempt, no-WAL-LSN dispatch - /// shape — a resolve reads the staged overlay and writes nothing. - /// - /// A capacity refusal returns [`DispatchOutcome::Deferred`]: the resolve - /// is parked for re-send and the txn stays in flight. A txn with no - /// `pending` entry returns [`DispatchOutcome::Failed`]. + /// A resolve reads the staged overlay and writes nothing, so no WAL LSN + /// rides on it. A capacity refusal returns [`DispatchOutcome::Deferred`]: + /// the resolve is parked for re-send and the txn stays in flight. A txn + /// with no `pending` entry returns [`DispatchOutcome::Failed`]. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn dispatch_calvin_resolve( &mut self, txn_id: TxnId, @@ -277,23 +187,18 @@ impl Scheduler { }; let tenant_id = pending.txn.tx_class.tenant_id; let database_id = pending.txn.tx_class.database_id; - let epoch = txn_id.epoch; - let position = txn_id.position; - let request_id = self.next_request_id(); - let plan = PhysicalPlan::Meta(MetaOp::CalvinResolve { epoch, position }); - // A resolve reads the staged overlay only; it writes no WAL record - // itself, so no committed LSN rides on this envelope. - let request = self.build_exempt_request(request_id, tenant_id, database_id, plan, None); - - // The resolve response re-enters the completion loop under the SAME - // txn_id, now in `AwaitingRedoResolve`, where `finish_redo_resolve` runs. + let plan = PhysicalPlan::Meta(MetaOp::CalvinResolve { + epoch: txn_id.epoch, + position: txn_id.position, + }); + let request = self.build_exempt_request(request_id, tenant_id, database_id, plan); self.dispatch_sequenced(txn_id, DispatchStep::Resolve, request) } } -/// The terminal error for a commit-resolution dispatch whose txn has no -/// `pending` entry to build the request from. +/// The terminal error for a dispatch whose txn has no `pending` entry to +/// build the request from. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn missing_pending_error( txn_id: TxnId, ) -> crate::Error { @@ -359,7 +264,7 @@ mod tests { assert_eq!( scheduler.pending.get(&txn_id).map(|p| p.commit_state), Some(CommitState::AwaitingRedoResolve), - "no flush is dispatched" + "no redo is proposed" ); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolution_dispatch.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolution_dispatch.rs deleted file mode 100644 index 228f0a329..000000000 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolution_dispatch.rs +++ /dev/null @@ -1,104 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Dispatch of staged Calvin flush/drop resolution operations. - -use nodedb_physical::physical_plan::PhysicalPlan; -use nodedb_physical::physical_plan::meta::MetaOp; - -use super::commit_redo::missing_pending_error; -use super::deferred::{DispatchOutcome, DispatchStep}; -use super::scheduler::Scheduler; -use crate::bridge::dispatch::JournalGroup; -use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; -use crate::types::Lsn; - -/// How a staged transaction resolves on this vShard. -pub(in crate::control::cluster::calvin::scheduler::driver::core) enum CommitResolution { - /// Install the committed redo record the scheduler appended at - /// `redo_lsn`. The flush scope holds its bytes. Both are empty when the - /// transaction wrote nothing on this vShard. - Flush { redo_lsn: Option }, - /// Discard the staged state under an abort verdict. - Drop, -} - -impl Scheduler { - /// Dispatch a flush or drop of a staged transaction's commit-pending buffer. - /// - /// A flush carries the redo record, the collections the local plans - /// write, and their materialized-sum targets, so the Data Plane installs - /// the record the way every committed transaction installs. - /// - /// A capacity refusal returns [`DispatchOutcome::Deferred`]: the flush or - /// drop is parked for re-send and the txn stays in flight. A txn with no - /// `pending` entry returns [`DispatchOutcome::Failed`]. The flush takes its - /// collections and sum targets from the scope derived at stage time. A - /// flush whose gates a halt released takes them back first, and fails when - /// a collection it writes no longer holds its planned incarnation. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn dispatch_commit_resolution( - &mut self, - txn_id: TxnId, - resolution: CommitResolution, - ) -> DispatchOutcome { - // A flush writes the collections: it holds their gates while it runs. - if matches!(resolution, CommitResolution::Flush { .. }) - && let Err(error) = self.regate(txn_id) - { - return DispatchOutcome::Failed(error); - } - let Some(pending) = self.pending.get_mut(&txn_id) else { - return DispatchOutcome::Failed(missing_pending_error(txn_id)); - }; - let tenant_id = pending.txn.tx_class.tenant_id; - let database_id = pending.txn.tx_class.database_id; - let event_source = - super::request::slice_event_source(&pending.txn.tx_class, &pending.flush_scope); - let commit_hlc = self - .cut_floors - .commit_hlc(pending.txn.epoch, pending.txn.epoch_system_ms); - let epoch = txn_id.epoch; - let position = txn_id.position; - let (plan, step, wal_lsn, journal) = match resolution { - CommitResolution::Flush { redo_lsn } => { - pending.flush_scope.sends = pending.flush_scope.sends.saturating_add(1); - let scope = &pending.flush_scope; - // The core stores the flush's append inputs beside its - // effects for boot. The record is whole at append, so no part - // follows it. - let journal = redo_lsn.map(|origin| JournalGroup { - origin, - collection: scope.collections.first().cloned().unwrap_or_default(), - apply_key: crate::wal::manager::NO_APPLY_KEY, - commit_hlc: Some(commit_hlc), - change_position: None, - }); - ( - PhysicalPlan::Meta(MetaOp::CalvinFlush { - epoch, - position, - redo: scope.redo.clone(), - collections: scope.collections.clone(), - sum_targets: scope.sum_targets.clone(), - }), - DispatchStep::Flush, - redo_lsn, - journal, - ) - } - CommitResolution::Drop => ( - PhysicalPlan::Meta(MetaOp::CalvinDrop { epoch, position }), - DispatchStep::Drop, - None, - None, - ), - }; - let request_id = self.next_request_id(); - let mut request = - self.build_exempt_request(request_id, tenant_id, database_id, plan, wal_lsn); - // The flush emits the transaction's events, under its own source and - // dated by its commit HLC. - request.event_source = event_source; - request.commit_hlc = Some(commit_hlc); - self.dispatch_sequenced_journalled(txn_id, step, request, journal) - } -} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/apply_tail.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/apply_tail.rs deleted file mode 100644 index 5445a93fa..000000000 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/apply_tail.rs +++ /dev/null @@ -1,674 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Commit tail: runs once a flush/drop response has returned, depositing the -//! applied result, marking the apply durable, recording write versions, and -//! proposing the `CompletionAck`. - -use crate::bridge::envelope::{ErrorCode, Response, Status}; -use crate::control::cluster::calvin::scheduler::driver::core::commit_resolution_dispatch::CommitResolution; -use crate::control::cluster::calvin::scheduler::driver::core::deferred::{ - DispatchOutcome, DispatchStep, -}; -use crate::control::cluster::calvin::scheduler::driver::core::halt::{ - HaltReason, HaltStep, error_response_text, -}; -use crate::control::cluster::calvin::scheduler::driver::core::owed::SchedulerProposal; -use crate::control::cluster::calvin::scheduler::driver::core::scheduler::Scheduler; -use crate::control::cluster::calvin::scheduler::driver::core::write_version_record::VersionRecord; -use crate::control::cluster::calvin::scheduler::driver::types::CommitState; -use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; -use crate::control::cluster::calvin::scheduler::metrics::infra_abort_reason; - -/// The most flushes one committed txn sends. Each refused install rolled -/// every write back, so a resend is safe. A refusal that outlasts the bound -/// halts the scheduler. -pub(in crate::control::cluster::calvin::scheduler::driver::core) const MAX_FLUSH_SENDS: u32 = 8; - -impl Scheduler { - /// Run the commit tail once a flush/drop response has returned. - /// - /// On a successful flush the full commit tail runs (deposit applied result, - /// `CalvinApplied` WAL + write-version recording, `CompletionAck`). On a - /// successful drop only the `CompletionAck` is proposed — the coordinator's - /// completion waiter still fires and the epoch advances, but nothing was - /// written so there is no result to deposit, no apply LSN, and no versions - /// to record. - /// - /// A flush refused with `RetryableRefusal` rolled its install back and - /// kept the staged buffer, so the scheduler sends the same flush again, up - /// to [`MAX_FLUSH_SENDS`] sends. Any other non-`Ok` flush, or a refusal - /// past the bound, halts the scheduler: a skipped flush tears the - /// committed txn on this replica. A non-`Ok` drop completes the txn: - /// under an abort verdict no replica writes anything. - pub(in crate::control::cluster::calvin::scheduler::driver::core) async fn finish_resolved_commit( - &mut self, - txn_id: TxnId, - response: Response, - committed: bool, - redo_lsn: Option, - ) { - if response.status != Status::Ok { - if committed && self.resend_refused_flush(txn_id, &response, redo_lsn) { - return; - } - if committed { - self.halt_apply( - txn_id, - HaltReason::FlushFailed, - HaltStep::Flush, - error_response_text("CalvinFlush", &response), - ); - return; - } - tracing::error!( - vshard_id = self.vshard_id, - epoch = txn_id.epoch, - position = txn_id.position, - "calvin: drop response was not Ok under an abort verdict; completing the \ - aborted txn, since no replica writes it" - ); - self.metrics.record_executor_error(); - self.metrics - .record_infra_abort(infra_abort_reason::IO_ERROR); - } - - let completed = if committed { - self.commit_apply_tail(txn_id, response, redo_lsn).await - } else { - // A dropped txn installed nothing, so its ack reports nothing. - // The coordinator's outcome waits for this ack, whatever the - // drop response said. - self.propose_sequencer_entry( - txn_id, - SchedulerProposal::CompletionAck { result: Vec::new() }, - ); - true - }; - // `false` means the txn stays pending and unapplied: the commit tail - // halted the scheduler, or its write-version record waits for capacity. - if completed { - self.metrics.record_completed(); - self.on_txn_complete(txn_id); - } - } - - /// Send the flush of `txn_id` again when the install refused it as - /// retryable and the send bound allows another. Returns whether the - /// refusal is handled: the flush went out again, or its dispatch failed - /// and the step's terminal handling ran. - fn resend_refused_flush( - &mut self, - txn_id: TxnId, - response: &Response, - redo_lsn: Option, - ) -> bool { - if !matches!( - response.error_code.as_deref(), - Some(ErrorCode::RetryableRefusal { .. }) - ) { - return false; - } - let sends = self - .pending - .get(&txn_id) - .map_or(MAX_FLUSH_SENDS, |pending| pending.flush_scope.sends); - if sends >= MAX_FLUSH_SENDS { - return false; - } - tracing::warn!( - vshard_id = self.vshard_id, - epoch = txn_id.epoch, - position = txn_id.position, - sends, - "calvin: the flush install was refused as retryable; sending it again" - ); - if let DispatchOutcome::Failed(error) = - self.dispatch_commit_resolution(txn_id, CommitResolution::Flush { redo_lsn }) - { - self.fail_dispatch_step(txn_id, DispatchStep::Flush, error); - } - true - } - - /// Deposit the applied result, durably mark the apply, record the apply's - /// write versions, and propose the `CompletionAck`. - /// - /// Returns `true` when the caller completes the txn. Returns `false` when - /// the txn stays pending, with its position unapplied: - /// - a WAL append, its fsync, or the write-version record failed, and the - /// scheduler halted; - /// - the write-version record waits for capacity. The txn then moves to - /// [`CommitState::AwaitingVersionRecord`] and completes once the record is sent. - /// - /// `redo_lsn` is `Some(lsn)` when a `TransactionRedo` record was already - /// WAL-appended for this commit's non-empty write set (`finish_redo_resolve`) - /// — that record already IS the durable applied marker, so only write - /// versions are recorded at it. `None` (an empty-ops staged commit, which - /// carries no redo record) appends a `CalvinApplied` marker here. - pub(in crate::control::cluster::calvin::scheduler::driver::core) async fn commit_apply_tail( - &mut self, - txn_id: TxnId, - response: Response, - redo_lsn: Option, - ) -> bool { - // The install folded materialized sums into target rows no redo - // sub-record names. The redo record's stamp carries the sum targets, - // so restart replay folds at the same LSN. - // Deposit the FULL applied Response (affected-count + watermark + any - // RETURNING rows) into the local sidecar BEFORE proposing the replicated - // CompletionAck. The ack fires the coordinator's completion oneshot on - // every sequencer member, so depositing first guarantees the result is - // present by the time the coordinator drains it — no lost result, no - // race. - // - // Gated on the PRIMARY-WRITE participant: any participant whose slice - // carries the user's non-edge DML (Document/KV/Vector/etc.), as opposed - // to the implicit graph-edge cleanup that dual-homes alongside it. A - // multi-collection cross-shard COMMIT has MANY primary-write - // participants — each a plain affected-count write — and they coalesce: - // the first applied response stands for the coordinator (which discards - // it for a COMMIT tag anyway), and the plain-write siblings do not - // conflict. Only a genuine cross-shard RETURNING union — two - // participants each carrying RETURNING rows — records `Conflict`. - // The primary slice's answer, its RETURNING rows or its affected - // count, and the timeseries install counts also ride the replicated - // CompletionAck, so a coordinator on a node with no replica of this - // vShard reads them. - let (has_primary_write, has_returning) = self - .pending - .get(&txn_id) - .map(|p| (p.has_primary_write, p.has_returning)) - .unwrap_or((false, false)); - let ack_result = self.ack_result_of( - &response, - crate::control::state::AckSlice { - primary_write: has_primary_write, - returning: has_returning, - }, - ); - if has_primary_write { - use std::collections::hash_map::Entry; - - let response = statement_reply(response); - - use crate::control::state::CalvinApplyResult; - - let key = nodedb_cluster::calvin::TxnId::new(txn_id.epoch, txn_id.position); - let vshard_id = self.vshard_id; - self.shared.calvin.apply_results.deposit_with(key, |entry| match entry { - Entry::Vacant(slot) => { - slot.insert(CalvinApplyResult::Single { - response, - has_returning, - }); - } - Entry::Occupied(mut slot) => { - // Derive both facts from the existing entry BEFORE any - // insert, so the immutable borrow does not outlive the - // mutable one. - let existing_returning = matches!( - slot.get(), - CalvinApplyResult::Single { - has_returning: true, - .. - } - ); - let already_conflict = matches!(slot.get(), CalvinApplyResult::Conflict); - - if already_conflict { - // A RETURNING union was already recorded; stays Conflict. - } else if has_returning && existing_returning { - // Two RETURNING-bearing participants for one Calvin txn: - // a cross-shard RETURNING union, which is unsupported. - // Record Conflict so the coordinator fails the statement - // loudly rather than returning one shard's partial rows. - tracing::error!( - epoch = txn_id.epoch, - position = txn_id.position, - vshard = vshard_id, - "two RETURNING-bearing participants for one Calvin txn — cross-shard \ - RETURNING union unsupported" - ); - slot.insert(CalvinApplyResult::Conflict); - } else if has_returning { - // The incoming participant carries the rows; the existing - // entry was a plain affected-count sibling. Rows win. - slot.insert(CalvinApplyResult::Single { - response, - has_returning: true, - }); - } else { - // Incoming is a plain write; keep the existing entry — a - // multi-collection cross-shard COMMIT coalesces (the - // coordinator discards it for a COMMIT tag anyway). - // The timeseries install counts of both merge, so the - // COMMIT reports every participant's apply count. - merge_install_counts(slot.get_mut(), &response); - } - } - }); - } - let applied_lsn = match redo_lsn { - // The TransactionRedo record already durably marks this apply — the - // SAME shard-local WAL-LSN space fast-path writes and read - // watermarks use. Record the apply's per-key write versions at it; - // no second (CalvinApplied) marker is written. - Some(lsn) => Some((lsn, self.record_calvin_write_versions(txn_id, lsn))), - None => match self - .shared - .wal - .appender(crate::wal::manager::NO_APPLY_KEY) - .with_commit_hlc(self.txn_commit_hlc(txn_id)) - .append_calvin_applied( - crate::types::VShardId::new(self.vshard_id), - txn_id.epoch, - txn_id.position, - ) { - // The CalvinApplied WAL LSN is the committed write-LSN for this - // apply — the SAME shard-local WAL-LSN space fast-path writes and - // read watermarks use. Record the apply's per-key write versions - // at it once it exists; it does not exist yet at dispatch time. - Ok(applied_lsn) => Some(( - applied_lsn, - self.record_calvin_write_versions(txn_id, applied_lsn), - )), - Err(e) => { - self.halt_apply( - txn_id, - HaltReason::WalAppendFailed, - HaltStep::AppliedMarker, - format!("CalvinApplied WAL append failed: {e}"), - ); - None - } - }, - }; - let Some((lsn, version_record)) = applied_lsn else { - // The apply cannot be acknowledged without a durable participant - // LSN: CDC and write-version consumers will otherwise observe a - // successful commit with no authoritative ordering point. The - // scheduler halted above. - return false; - }; - if version_record == VersionRecord::Halted { - // Read-set validation reads the lost versions. The scheduler - // halted, and the txn stays pending and unapplied. - return false; - } - // The record is whole at append: the flush reports no rows, so no - // part follows it. A record split over the WAL record limit is - // durable once its last continuation is. - let durable_through = self - .pending - .get(&txn_id) - .and_then(|pending| pending.redo_records.as_ref()) - .and_then(|records| records.last_lsn()) - .map_or(lsn, |last| last.max(lsn)); - // Control change-stream events are distinct from Data-Plane - // WriteEvents. Every replica publishes the rows this participant's - // redo installs at the transaction's sequencer position, which every - // replica shares; the vShard's leader forwards them to the nodes - // that do not replicate it. The publish journals them durably before - // the applied marker's fsync below, so a restart that finds the - // position applied also finds its changes. - if let Some(pending) = self.pending.get_mut(&txn_id) { - let calvin = crate::control::server::dispatch_utils::CalvinApply { - tenant_id: pending.txn.tx_class.tenant_id, - database_id: pending.txn.tx_class.database_id, - vshard: self.vshard_id, - sequencer_epoch: txn_id.epoch, - position: txn_id.position, - commit_hlc: self - .cut_floors - .commit_hlc(pending.txn.epoch, pending.txn.epoch_system_ms), - }; - let change_sets = std::mem::take(&mut pending.change_sets); - crate::control::server::dispatch_utils::publish_calvin_change_sets( - &self.shared, - calvin, - change_sets, - lsn, - ); - } - // The record at `lsn` is this position's only applied marker. An - // append only buffers it, so it is durable before the mark and the - // ack: a restart that lost it will take the position for unapplied, - // run the transaction again, and never settle its ack. The wait joins - // the WAL group commit, so concurrent Calvin commits share one fsync. - if let Err(e) = self.shared.wal.wait_durable(durable_through).await { - self.halt_apply( - txn_id, - HaltReason::WalAppendFailed, - HaltStep::AppliedMarker, - format!("applied marker fsync at lsn {} failed: {e}", lsn.as_u64()), - ); - return false; - } - if let Some(origin) = redo_lsn { - self.note_flush_settled(origin); - self.record_applied_key(txn_id); - } - // The commit's mark lands before the ack, as a write through the - // funnel records its mark before its response returns. - self.record_calvin_write_mark(txn_id); - self.propose_sequencer_entry( - txn_id, - SchedulerProposal::CompletionAck { result: ack_result }, - ); - if version_record == VersionRecord::Parked { - // The txn keeps its locks until the record is sent, so no later - // txn validates a read before the versions land. - if let Some(pending) = self.pending.get_mut(&txn_id) { - pending.commit_state = CommitState::AwaitingVersionRecord; - } - return false; - } - true - } -} - -impl Scheduler { - /// Record the dedup key this participant's redo record carries, once the - /// record is durable and installed. Every replica records it, as a - /// Raft-applied redo's key is, so a request or trigger body sent again to - /// any replica applies once. - fn record_applied_key(&self, txn_id: TxnId) { - let Some(pending) = self.pending.get(&txn_id) else { - return; - }; - let tx_class = &pending.txn.tx_class; - if tx_class.applied_key.is_empty() - || tx_class.applied_key_home().ok().flatten() != Some(self.vshard_id) - { - return; - } - let Some(dedup) = self.shared.cross_shard_dedup.get() else { - return; - }; - match zerompk::from_msgpack::(&tx_class.applied_key) { - Ok(key) => { - if let Err(error) = dedup.record_applied(&key) { - tracing::error!( - vshard_id = self.vshard_id, - origin = %key.origin, - %error, - "calvin: dedup key held in memory only; the WAL restores it on restart" - ); - } - } - Err(error) => tracing::error!( - vshard_id = self.vshard_id, - %error, - "calvin: the applied key does not decode; it is not recorded" - ), - } - } - - /// The report the flush `response` owes the coordinator, encoded for the - /// replicated `CompletionAck`: the apply's timeseries install counts, and - /// the rows of a `returning` slice. The rows are bounded by the query - /// result limit and by the ack's frame budget, whichever is lower. - fn ack_result_of( - &self, - response: &Response, - slice: crate::control::state::AckSlice, - ) -> Vec { - let limit = self - .shared - .tuning - .network - .max_query_result_bytes - .min(crate::control::state::ACK_ROWS_FRAME_BUDGET); - crate::control::state::CalvinAckResult::of(response, slice, limit) - .to_bytes() - .unwrap_or_else(|error| { - tracing::warn!( - vshard_id = self.vshard_id, - %error, - "calvin: the completion ack carries no report" - ); - Vec::new() - }) - } -} - -/// The response the statement drains. A flush whose install succeeded but -/// whose reply failed to render answers `Ok` with the render error in -/// `error_code`. The statement reports that error. -/// Fold the timeseries install counts `incoming` answers into the plain -/// result `held`. A RETURNING result keeps its rows untouched. -fn merge_install_counts(held: &mut crate::control::state::CalvinApplyResult, incoming: &Response) { - let crate::control::state::CalvinApplyResult::Single { - response, - has_returning: false, - } = held - else { - return; - }; - if let Some(merged) = crate::engine::timeseries::install_counts::merge_count_payloads( - response.payload.as_bytes(), - incoming.payload.as_bytes(), - ) { - response.payload = crate::bridge::envelope::Payload::from_vec(merged); - } -} - -fn statement_reply(mut response: Response) -> Response { - if response.status == Status::Ok && response.error_code.is_some() { - response.status = Status::Error; - response.payload = crate::bridge::envelope::Payload::empty(); - } - response -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ - error_response, scheduler_with_pending, staged_response, - }; - use crate::control::cluster::calvin::scheduler::driver::types::CommitState; - - fn internal_error() -> Response { - error_response(ErrorCode::Internal { - detail: "core failed".to_string(), - }) - } - - fn flushed_with_counts(collection: &str, accepted: u64, rejected: u64) -> Response { - use crate::engine::timeseries::install_counts::{TsInstallCount, TsInstallCounts}; - let mut response = staged_response(Status::Ok, None); - response.payload = crate::bridge::envelope::Payload::from_vec( - TsInstallCounts::new(vec![TsInstallCount { - collection: collection.into(), - accepted, - rejected, - }]) - .to_bytes() - .expect("encode install counts"), - ); - response - } - - /// Two plain participants of one Calvin transaction each installed a - /// timeseries batch. The coordinator's result carries both installs' - /// apply counts, so COMMIT reports every participant's rejected rows. A - /// RETURNING result keeps its rows. - #[test] - fn plain_participants_merge_their_install_counts() { - use crate::control::state::CalvinApplyResult; - use crate::engine::timeseries::install_counts::TsInstallCounts; - let mut held = CalvinApplyResult::Single { - response: flushed_with_counts("cpu", 2, 1), - has_returning: false, - }; - merge_install_counts(&mut held, &flushed_with_counts("mem", 3, 2)); - let CalvinApplyResult::Single { response, .. } = &held else { - panic!("a merged result stays single"); - }; - let totals = TsInstallCounts::from_payload(response.payload.as_bytes()) - .expect("install counts") - .by_collection(); - assert_eq!(totals.get("cpu"), Some(&(2, 1))); - assert_eq!(totals.get("mem"), Some(&(3, 2))); - - let rows = crate::bridge::envelope::Payload::from_vec(vec![0x90]); - let mut returning = CalvinApplyResult::Single { - response: Response { - payload: rows, - ..flushed_with_counts("cpu", 1, 0) - }, - has_returning: true, - }; - merge_install_counts(&mut returning, &flushed_with_counts("mem", 1, 1)); - let CalvinApplyResult::Single { response, .. } = &returning else { - panic!("a RETURNING result stays single"); - }; - assert_eq!(response.payload.as_bytes(), &[0x90]); - } - - /// A flush that returns an error under a COMMIT verdict holds the txn - /// unapplied and halts: a second flush will apply nothing. - #[tokio::test] - async fn flush_error_response_holds_committed_txn_unapplied() { - let txn_id = TxnId::new(9, 2); - let (mut scheduler, _dir) = scheduler_with_pending( - txn_id, - CommitState::AwaitingResolve { - committed: true, - redo_lsn: None, - }, - ); - - scheduler - .finish_resolved_commit(txn_id, internal_error(), true, None) - .await; - - assert!(!scheduler.applied.is_applied(9, 2)); - assert!(scheduler.pending.contains_key(&txn_id)); - assert_eq!( - scheduler.apply_halt().map(|h| h.reason), - Some(HaltReason::FlushFailed) - ); - assert!(scheduler.shared.sequencer_halt.apply_halt().is_halted()); - } - - /// A drop that returns an error under an abort verdict still completes - /// the txn and owes its ack: no replica writes an aborted txn. - #[tokio::test] - async fn drop_error_response_under_abort_completes_txn() { - let txn_id = TxnId::new(9, 2); - let (mut scheduler, _dir) = scheduler_with_pending( - txn_id, - CommitState::AwaitingResolve { - committed: false, - redo_lsn: None, - }, - ); - - scheduler - .finish_resolved_commit(txn_id, internal_error(), false, None) - .await; - - assert!(scheduler.applied.is_applied(9, 2)); - assert!(!scheduler.pending.contains_key(&txn_id)); - assert!(!scheduler.is_apply_halted()); - assert!(scheduler.owed.contains_key(&( - txn_id, - crate::control::cluster::calvin::scheduler::driver::core::owed::OwedKind::CompletionAck - ))); - } - fn retryable_refusal() -> Response { - error_response(ErrorCode::RetryableRefusal { - reason: "install rolled back".to_string(), - }) - } - - /// A flush refused as retryable reaches the Data Plane again with the - /// same redo record, and the txn stays pending and unhalted. - #[tokio::test] - async fn retryable_flush_refusal_resends_the_same_flush() { - use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ - await_data_plane_request, build_test_scheduler_with_data_side, make_sequenced_txn, - staged_pending, - }; - use nodedb_physical::physical_plan::PhysicalPlan; - use nodedb_physical::physical_plan::meta::MetaOp; - - let txn_id = TxnId::new(9, 2); - let registry = nodedb_cluster::calvin::CalvinCompletionRegistry::new_detached(); - let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(7, registry); - let mut pending = staged_pending(make_sequenced_txn(9, 2), txn_id); - pending.commit_state = CommitState::AwaitingResolve { - committed: true, - redo_lsn: None, - }; - pending.flush_scope.redo = vec![7, 7, 7]; - pending.flush_scope.sends = 1; - scheduler.pending.insert(txn_id, pending); - - scheduler - .finish_resolved_commit(txn_id, retryable_refusal(), true, None) - .await; - - assert!(!scheduler.is_apply_halted()); - assert!(!scheduler.applied.is_applied(9, 2)); - assert_eq!( - scheduler.pending.get(&txn_id).map(|p| p.flush_scope.sends), - Some(2) - ); - assert!( - await_data_plane_request(&mut data_side, |plan| matches!( - plan, - PhysicalPlan::Meta(MetaOp::CalvinFlush { epoch: 9, position: 2, redo, .. }) - if redo == &vec![7, 7, 7] - )) - .await, - "the resent flush carries the same redo record" - ); - } - - /// A retryable refusal past the send bound halts like any flush error. - #[tokio::test] - async fn retryable_flush_refusal_past_the_bound_halts() { - let txn_id = TxnId::new(9, 2); - let (mut scheduler, _dir) = scheduler_with_pending( - txn_id, - CommitState::AwaitingResolve { - committed: true, - redo_lsn: None, - }, - ); - if let Some(pending) = scheduler.pending.get_mut(&txn_id) { - pending.flush_scope.sends = MAX_FLUSH_SENDS; - } - - scheduler - .finish_resolved_commit(txn_id, retryable_refusal(), true, None) - .await; - - assert!(scheduler.pending.contains_key(&txn_id)); - assert_eq!( - scheduler.apply_halt().map(|h| h.reason), - Some(HaltReason::FlushFailed) - ); - } - - /// A render error on an installed flush reaches the statement as a typed - /// error, and the txn completes. - #[test] - fn a_render_error_on_an_installed_flush_becomes_the_statement_error() { - let mut response = error_response(ErrorCode::Internal { - detail: "render".to_string(), - }); - response.status = Status::Ok; - - let reply = statement_reply(response); - - assert_eq!(reply.status, Status::Error); - assert!(matches!( - reply.error_code.as_deref(), - Some(ErrorCode::Internal { .. }) - )); - } -} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/drop_tail.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/drop_tail.rs new file mode 100644 index 000000000..2d1d3786c --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/drop_tail.rs @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The end of a staged slice that commits no log entry: an aborted txn, or a +//! committed slice with no local write. Its drop answered, so it owes an +//! empty `CompletionAck` and completes. + +use crate::bridge::envelope::{Response, Status}; +use crate::control::cluster::calvin::scheduler::driver::core::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; +use crate::control::cluster::calvin::scheduler::metrics::infra_abort_reason; + +impl Scheduler { + /// Complete `txn_id` once its drop answered. + /// + /// A drop that did not answer `Ok` still completes the txn: no replica + /// writes it, whatever the drop did on this core. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn finish_drop( + &mut self, + txn_id: TxnId, + response: &Response, + ) { + if response.status != Status::Ok { + tracing::error!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + error = ?response.error_code.as_deref(), + "calvin: drop answered with an error; completing the txn, since no replica \ + writes it" + ); + self.metrics.record_executor_error(); + self.metrics + .record_infra_abort(infra_abort_reason::IO_ERROR); + } + self.complete_without_entry(txn_id); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bridge::envelope::ErrorCode; + use crate::control::cluster::calvin::scheduler::driver::core::owed::OwedKind; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + error_response, scheduler_with_pending, staged_response, + }; + use crate::control::cluster::calvin::scheduler::driver::types::CommitState; + + /// A drop that answers `Ok` completes the txn and owes its ack. + #[tokio::test] + async fn an_answered_drop_completes_the_txn_and_owes_its_ack() { + let txn_id = TxnId::new(9, 1); + let (mut scheduler, _dir) = scheduler_with_pending(txn_id, CommitState::AwaitingDrop); + + scheduler.finish_drop(txn_id, &staged_response(Status::Ok, None)); + + assert!(scheduler.applied.is_applied(9, 1)); + assert!(scheduler.ledger.is_applied(9, 1), "no log entry marks it"); + assert!(!scheduler.pending.contains_key(&txn_id)); + assert!( + scheduler + .owed + .contains_key(&(txn_id, OwedKind::CompletionAck)) + ); + } + + /// A drop that returns an error still completes the txn and owes its + /// ack: no replica writes a txn that ends with no log entry. + #[tokio::test] + async fn a_drop_error_still_completes_the_txn() { + let txn_id = TxnId::new(9, 2); + let (mut scheduler, _dir) = scheduler_with_pending(txn_id, CommitState::AwaitingDrop); + + scheduler.finish_drop( + txn_id, + &error_response(ErrorCode::Internal { + detail: "core failed".to_string(), + }), + ); + + assert!(scheduler.applied.is_applied(9, 2)); + assert!(!scheduler.pending.contains_key(&txn_id)); + assert!(!scheduler.is_apply_halted()); + assert!( + scheduler + .owed + .contains_key(&(txn_id, OwedKind::CompletionAck)) + ); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/flush_parts.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/flush_parts.rs deleted file mode 100644 index 7a830fc08..000000000 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/flush_parts.rs +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! The stored write set of a committed Calvin flush. -//! -//! A committed Calvin record is whole at append: its flush reports no rows, -//! so no part follows it. The core still runs the flush journalled, which -//! stores the flush's append inputs beside its effects for boot. Once the -//! record is durable, the core drops what it stored. - -use crate::control::cluster::calvin::scheduler::driver::core::scheduler::Scheduler; -use crate::types::{Lsn, VShardId}; - -impl Scheduler { - /// Note that the record at `origin` is durable, so the core drops the - /// write set it stored. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn note_flush_settled( - &self, - origin: Lsn, - ) { - let vshard_id = VShardId::new(self.vshard_id); - match self.shared.dispatcher.lock() { - Ok(mut d) => d.note_write_set_settled(vshard_id, origin), - Err(poisoned) => poisoned - .into_inner() - .note_write_set_settled(vshard_id, origin), - } - } -} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/mod.rs index 76ec99820..309b4965d 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/mod.rs @@ -1,18 +1,15 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Verdict-driven commit resolution for staged static Calvin transactions. +//! Verdict-driven commit resolution for a staged Calvin transaction on the +//! data-group leader, and verdict handling for a follower's held txn. //! -//! A static Calvin dispatch STAGES its transaction on the Data Plane (validate -//! the read-set + buffer the plans, no base mutation). Its executor response -//! carries the local commit vote on `stage_vote`. This module drives the -//! final step: dispatch a flush (commit, after `commit_redo` has WAL-appended -//! the resolved `TransactionRedo`) or drop (abort) of the staged buffer, wait -//! for its response, then run the commit tail (deposit applied result, record -//! write versions — plus a `CalvinApplied` WAL fallback when no redo record -//! was appended — propose `CompletionAck`) for a flush, or ack-only for a -//! drop. +//! A static or active dispatch STAGES its transaction on the Data Plane +//! (validate the read-set + buffer the plans, no base mutation). Its executor +//! response carries the local commit vote on `stage_vote`. The leader votes, +//! waits for the global verdict, then resolves a committed write slice into +//! its redo and proposes it (see `super::commit_redo`), or drops the staged +//! state of a slice that ends with no log entry. -mod apply_tail; -mod flush_parts; +mod drop_tail; mod verdict; mod vote; diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs index ecc90c5f8..334f34e86 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs @@ -1,42 +1,46 @@ // SPDX-License-Identifier: BUSL-1.1 //! Resume-on-verdict, verdict-signal handling, and the stall re-probe sweep -//! for a staged Calvin transaction parked on the cross-shard commit barrier. +//! for a granted Calvin transaction waiting on the cross-shard commit +//! barrier. use std::sync::atomic::Ordering; use std::time::Instant; use nodedb_cluster::calvin::VerdictSignal; -use crate::control::cluster::calvin::scheduler::driver::core::commit_resolution_dispatch::CommitResolution; use crate::control::cluster::calvin::scheduler::driver::core::deferred::{ DispatchOutcome, DispatchStep, }; use crate::control::cluster::calvin::scheduler::driver::core::halt::{HaltReason, HaltStep}; +use crate::control::cluster::calvin::scheduler::driver::core::owed::SchedulerProposal; +use crate::control::cluster::calvin::scheduler::driver::core::process::LedgerMark; use crate::control::cluster::calvin::scheduler::driver::core::scheduler::Scheduler; use crate::control::cluster::calvin::scheduler::driver::types::CommitState; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; impl Scheduler { - /// Resume a txn parked in [`CommitState::AwaitingVerdict`] once the durable - /// GLOBAL verdict is known: dispatch its flush (commit) or drop (abort). + /// Act on the durable GLOBAL verdict of `txn_id`. /// - /// `committed` is the authoritative cross-shard verdict — NOT this shard's - /// local vote. On commit, dispatches `MetaOp::CalvinResolve` and moves the - /// txn to [`CommitState::AwaitingRedoResolve`] (the resolved redo is - /// WAL-appended and the flush dispatched from [`Self::finish_redo_resolve`]). - /// On abort, dispatches the drop directly and moves the txn to - /// [`CommitState::AwaitingResolve`]. Bumps the flushed / dropped counter. The - /// commit tail runs later in [`Self::finish_resolved_commit`], once the - /// flush/drop response arrives. + /// `committed` is the authoritative cross-shard verdict, never this + /// shard's local vote. /// - /// Double-resume guard: the verdict push and the probe-on-park (and the - /// stall re-probe sweep) can all fire for one txn, so this first confirms the - /// txn is still `Some(AwaitingVerdict)` — if it already transitioned out - /// (resolve/drop dispatched, or completed), this is a no-op. This guarantees - /// the flush/drop is dispatched exactly once. + /// A leader's txn parked in [`CommitState::AwaitingVerdict`]: + /// - abort: dispatch the drop of its staged state; + /// - COMMIT for a slice with no write: drop the staged state too, since + /// nothing installs; + /// - COMMIT for a write slice: resolve its staged post-images, at once, + /// or at its turn for a whole-collection resolve. The resolved redo is + /// proposed from [`Self::finish_redo_resolve`]. /// - /// A COMMIT verdict for a txn this replica failed to stage halts the + /// A follower's [`CommitState::Following`] txn completes at an abort or + /// at a COMMIT for a slice with no write: no log entry follows either. + /// A COMMIT for a write slice waits for the slice's redo to apply. + /// + /// Any other state already left the barrier, so a duplicate push, probe + /// or sweep is a no-op. + /// + /// A COMMIT verdict for a txn this leader failed to stage halts the /// scheduler: the txn stays parked with its locks, its stall deadline /// cleared, and its position unapplied. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn resume_on_verdict( @@ -44,15 +48,30 @@ impl Scheduler { txn_id: TxnId, committed: bool, ) { - // Guard: only a still-parked txn resumes. Mirrors `handle_completion`'s - // state-match so a duplicate push/probe/timeout is idempotent. - if !matches!( - self.pending.get(&txn_id).map(|p| p.commit_state), - Some(CommitState::AwaitingVerdict) - ) { + let Some((state, writes)) = self + .pending + .get(&txn_id) + .map(|p| (p.commit_state, p.scope.writes)) + else { return; + }; + match state { + CommitState::Following => { + if !committed || !writes { + self.complete_without_entry(txn_id); + } + } + CommitState::AwaitingVerdict => self.resume_parked(txn_id, committed, writes), + CommitState::Staged + | CommitState::AwaitingResolveTurn + | CommitState::AwaitingRedoResolve + | CommitState::AwaitingRedoApply { .. } + | CommitState::AwaitingDrop => {} } + } + /// Resume a leader's txn parked on the barrier. + fn resume_parked(&mut self, txn_id: TxnId, committed: bool, writes: bool) { if committed && let Some(pending) = self.pending.get_mut(&txn_id) && let Some(stage_error) = pending.stage_error.clone() @@ -62,87 +81,89 @@ impl Scheduler { txn_id, HaltReason::LocalStageFailed, HaltStep::Stage, - format!("COMMIT verdict for a txn this replica did not stage: {stage_error}"), + format!("COMMIT verdict for a txn this leader did not stage: {stage_error}"), ); return; } - - // A slice that truncates rows resolves at its turn, once every lower - // txn of this vShard finished. - if committed - && let Some(pending) = self.pending.get_mut(&txn_id) - && pending.flush_scope.resolve_at_turn - { - pending.commit_state = CommitState::AwaitingResolveTurn; + if let Some(pending) = self.pending.get_mut(&txn_id) { + // No longer parked: clear the stall deadline. pending.verdict_deadline = None; - self.shared - .calvin - .counters - .commits_flushed - .fetch_add(1, Ordering::Relaxed); - self.pump_flush_turn(); - return; } - - let (outcome, step) = if committed { - // Resolve the staged post-images into a replayable `RedoRecord` - // first; the redo is WAL-appended (in `finish_redo_resolve`) before - // the flush is dispatched, restoring restart durability for this - // vShard's slice of a multi-shard Calvin commit. - (self.dispatch_calvin_resolve(txn_id), DispatchStep::Resolve) + let counter = if committed { + &self.shared.calvin.counters.commits_flushed } else { - ( - self.dispatch_commit_resolution(txn_id, CommitResolution::Drop), - DispatchStep::Drop, - ) + &self.shared.calvin.counters.commits_dropped }; - if let DispatchOutcome::Failed(error) = outcome { - // Terminal resolve/drop refusal: the scheduler halts and holds - // the txn parked with its locks and staged buffer. The cleared - // deadline keeps the stall sweep from re-sending it. + counter.fetch_add(1, Ordering::Relaxed); + + if !committed || !writes { + self.drop_staged_slice(txn_id); + return; + } + // A slice that truncates rows resolves at its turn, once every lower + // txn of this vShard finished. + let at_turn = self + .pending + .get(&txn_id) + .is_some_and(|pending| pending.scope.resolve_at_turn); + if at_turn { if let Some(pending) = self.pending.get_mut(&txn_id) { - pending.verdict_deadline = None; + pending.commit_state = CommitState::AwaitingResolveTurn; } - self.fail_dispatch_step(txn_id, step, error); + self.pump_resolve_turn(); return; } + self.start_resolve(txn_id); + } - // Sent or parked for re-send at capacity: either way the txn awaits - // this step's response, so a duplicate verdict push or probe is a - // no-op under the guard above. + /// Dispatch the resolve of a committed slice, and wait for its answer. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn start_resolve( + &mut self, + txn_id: TxnId, + ) { if let Some(pending) = self.pending.get_mut(&txn_id) { - pending.commit_state = if committed { - CommitState::AwaitingRedoResolve - } else { - CommitState::AwaitingResolve { - committed: false, - redo_lsn: None, - } - }; - // No longer parked: clear the stall deadline. - pending.verdict_deadline = None; + pending.commit_state = CommitState::AwaitingRedoResolve; } + // Sent or parked for re-send at capacity: either way the txn awaits + // the resolve's answer. + if let DispatchOutcome::Failed(error) = self.dispatch_calvin_resolve(txn_id) { + self.fail_dispatch_step(txn_id, DispatchStep::Resolve, error); + } + } - if committed { - self.shared - .calvin - .counters - .commits_flushed - .fetch_add(1, Ordering::Relaxed); - } else { - self.shared - .calvin - .counters - .commits_dropped - .fetch_add(1, Ordering::Relaxed); + /// Dispatch the drop of a staged slice that ends with no log entry, and + /// wait for its answer. + fn drop_staged_slice(&mut self, txn_id: TxnId) { + if let Some(pending) = self.pending.get_mut(&txn_id) { + pending.commit_state = CommitState::AwaitingDrop; + } + if let DispatchOutcome::Failed(error) = self.dispatch_drop(txn_id, DispatchStep::Drop) { + // Terminal refusal: the scheduler halts and holds the txn with + // its locks and staged buffer. + self.fail_dispatch_step(txn_id, DispatchStep::Drop, error); } } + /// Complete `txn_id`, which ends with no log entry on this vShard. It + /// owes an empty `CompletionAck`: the coordinator's outcome waits for + /// every participant's. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn complete_without_entry( + &mut self, + txn_id: TxnId, + ) { + self.propose_sequencer_entry( + txn_id, + SchedulerProposal::CompletionAck { result: Vec::new() }, + ); + self.metrics.record_completed(); + self.on_txn_complete(txn_id, LedgerMark::Terminal); + } + /// Handle a pushed [`VerdictSignal`] from this node's completion registry. /// - /// Matches the signal to the parked txn by `(epoch, position)` and resumes - /// it. A signal for a txn this scheduler does not host, or one that already - /// resumed, is a harmless no-op (the double-resume guard covers the latter). + /// Matches the signal to the txn by `(epoch, position)` and resumes it. A + /// signal for a txn this scheduler does not host, or one that already + /// resumed, is a harmless no-op. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn handle_verdict_signal( &mut self, signal: VerdictSignal, @@ -159,7 +180,7 @@ impl Scheduler { /// emit a stall metric + warning, and re-arm the deadline so the warning is /// rate-limited rather than per-iteration. It NEVER releases locks and NEVER /// unilaterally aborts: a participant cannot know whether a peer already - /// flushed a COMMIT, so aborting one side while a peer committed will tear + /// committed, so aborting one side while a peer committed will tear /// the transaction. The verdict is guaranteed to arrive eventually — a /// post-failover leader re-aggregates the replicated votes (seeded on every /// replica) into the same verdict — so waiting is always the safe action. @@ -202,7 +223,7 @@ impl Scheduler { vote_owed, "calvin: staged txn still awaiting the cross-shard verdict past its stall \ deadline; HOLDING locks and waiting (never aborting — a peer may have already \ - flushed a commit). The verdict is guaranteed to arrive." + committed). The verdict is guaranteed to arrive." ); if let Some(pending) = self.pending.get_mut(&txn_id) { pending.verdict_deadline = Some(now + self.config.verdict_stall_warn()); @@ -227,10 +248,11 @@ mod tests { use crate::bridge::envelope::ErrorCode; use crate::bridge::envelope::{Payload, StageVote, Status}; use crate::control::cluster::calvin::scheduler::driver::core::halt::HaltReason; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::lead_data_group; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ - await_data_plane_request, build_test_scheduler_with_data_side, error_response, - fill_tenant_inflight, make_sequenced_txn, release_filler, spawn_scheduler_loop, - staged_pending, staged_response, + await_data_plane_request, build_test_scheduler, build_test_scheduler_with_data_side, + error_response, fill_tenant_inflight, following_pending, make_sequenced_txn, + release_filler, spawn_scheduler_loop, staged_pending, staged_response, }; use crate::control::state::SharedState; use crate::types::RequestId; @@ -264,8 +286,7 @@ mod tests { } } - /// An Ok resolve response whose redo record carries no ops, so the flush - /// dispatch follows at once with no WAL append. + /// An Ok resolve response whose redo record carries no ops. fn empty_redo_response() -> crate::bridge::envelope::Response { let redo = RedoRecord { version: 1, @@ -306,24 +327,28 @@ mod tests { ); } - /// A refused flush dispatch after the redo resolves does not complete the - /// txn: its position stays unapplied and its pending entry stays. + /// A resolved redo is proposed to the data group, and the txn waits for + /// its apply: its position stays unapplied and its pending entry stays. #[tokio::test] - async fn resolved_redo_flush_refused_at_capacity_does_not_complete_txn() { + async fn a_resolved_redo_is_proposed_and_waits_for_its_apply() { let txn_id = TxnId::new(14, 2); - let mut parked = parked_at_capacity(txn_id, CommitState::AwaitingRedoResolve); - let scheduler = &mut parked.scheduler; + let (mut scheduler, _dir) = build_test_scheduler(7); + lead_data_group(&mut scheduler); + let mut pending = staged_pending(make_sequenced_txn(14, 2), txn_id); + pending.commit_state = CommitState::AwaitingRedoResolve; + scheduler.pending.insert(txn_id, pending); scheduler.finish_redo_resolve(txn_id, empty_redo_response()); + assert!(matches!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingRedoApply { proposed: Some(_) }) + )); assert!( !scheduler.applied.is_applied(14, 2), - "a refused flush must not mark the position applied" - ); - assert!( - scheduler.pending.contains_key(&txn_id), - "a refused flush must keep the txn's pending entry" + "a proposed redo must not mark the position applied" ); + assert!(!scheduler.is_apply_halted()); } /// Under an ABORT verdict, a refused drop dispatch does not complete the @@ -381,10 +406,10 @@ mod tests { ); } - /// Once a Data Plane response frees tenant capacity, a refused flush + /// Once a Data Plane response frees tenant capacity, a refused drop /// reaches the Data Plane. #[tokio::test] - async fn refused_flush_reaches_data_plane_after_capacity_frees() { + async fn refused_drop_reaches_data_plane_after_capacity_frees() { let txn_id = TxnId::new(14, 2); let ParkedAtCapacity { mut scheduler, @@ -392,19 +417,18 @@ mod tests { mut data_side, shared, fillers, - } = parked_at_capacity(txn_id, CommitState::AwaitingRedoResolve); + } = parked_at_capacity(txn_id, CommitState::AwaitingVerdict); - scheduler.finish_redo_resolve(txn_id, empty_redo_response()); + scheduler.resume_on_verdict(txn_id, false); let running = spawn_scheduler_loop(scheduler); release_filler(&shared, &mut data_side, fillers[0]); let arrived = await_data_plane_request(&mut data_side, |plan| { matches!( plan, - PhysicalPlan::Meta(MetaOp::CalvinFlush { + PhysicalPlan::Meta(MetaOp::CalvinDrop { epoch: 14, position: 2, - .. }) ) }) @@ -413,14 +437,14 @@ mod tests { assert!( arrived, - "the refused flush must reach the Data Plane once capacity frees" + "the refused drop must reach the Data Plane once capacity frees" ); } /// A false vote from either participant makes the only global verdict abort; /// applying that durable verdict broadcasts the abort to every parked local /// participant. The scheduler's `resume_on_verdict(false)` then dispatches a - /// drop, never a resolve/flush, on each recipient. + /// drop, never a resolve, on each recipient. #[tokio::test] async fn two_participant_false_vote_broadcasts_global_abort_to_every_scheduler() { let registry = CalvinCompletionRegistry::new_detached(); @@ -501,10 +525,7 @@ mod tests { .pending .get(&txn_id) .map(|pending| pending.commit_state), - Some(CommitState::AwaitingResolve { - committed: false, - redo_lsn: None - }) + Some(CommitState::AwaitingDrop) )); let request = data_side .request_rx @@ -521,16 +542,12 @@ mod tests { } } - /// A follower scheduler whose stage failed, parked on the verdict barrier. - fn follower_with_failed_stage( + /// A leader scheduler whose stage failed, parked on the verdict barrier. + fn leader_with_failed_stage( txn_id: TxnId, ) -> (Scheduler, tempfile::TempDir, CoreChannelDataSide) { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, dir, data_side) = build_test_scheduler_with_data_side(7, registry); - assert!( - !scheduler.is_group_leader(), - "the fixture hosts no data group" - ); scheduler.pending.insert( txn_id, staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id), @@ -544,13 +561,14 @@ mod tests { (scheduler, dir, data_side) } - /// A COMMIT verdict for a txn this follower failed to stage halts the - /// scheduler: the leader staged and voted commit, so this replica cannot - /// apply it. No resolve is dispatched, and the stall sweep stops. + /// A COMMIT verdict for a txn this leader failed to stage halts the + /// scheduler: an earlier leader staged and voted commit, so this leader + /// holds nothing to resolve. No resolve is dispatched, and the stall + /// sweep stops. #[tokio::test] async fn commit_verdict_after_local_stage_error_halts_unapplied() { let txn_id = TxnId::new(14, 2); - let (mut scheduler, _dir, mut data_side) = follower_with_failed_stage(txn_id); + let (mut scheduler, _dir, mut data_side) = leader_with_failed_stage(txn_id); scheduler.resume_on_verdict(txn_id, true); @@ -571,12 +589,12 @@ mod tests { ); } - /// An abort verdict for a txn this replica failed to stage drops it as + /// An abort verdict for a txn this leader failed to stage drops it as /// usual: every replica reaches the same abort. #[tokio::test] async fn abort_verdict_after_local_stage_error_drops_without_halting() { let txn_id = TxnId::new(14, 2); - let (mut scheduler, _dir, mut data_side) = follower_with_failed_stage(txn_id); + let (mut scheduler, _dir, mut data_side) = leader_with_failed_stage(txn_id); scheduler.resume_on_verdict(txn_id, false); @@ -593,4 +611,38 @@ mod tests { }) )); } + + /// A follower's txn completes at an abort verdict with no log entry: it + /// owes its ack and marks its position applied. A COMMIT verdict for a + /// write slice leaves it waiting for its redo. + #[tokio::test] + async fn a_following_txn_completes_at_an_abort_and_waits_at_a_commit() { + use crate::control::cluster::calvin::scheduler::driver::core::owed::OwedKind; + let (mut scheduler, _dir) = build_test_scheduler(7); + let aborted = TxnId::new(14, 2); + let committed = TxnId::new(15, 0); + for txn_id in [aborted, committed] { + scheduler.pending.insert( + txn_id, + following_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id), + ); + } + + scheduler.resume_on_verdict(aborted, false); + scheduler.resume_on_verdict(committed, true); + + assert!(!scheduler.pending.contains_key(&aborted)); + assert!(scheduler.applied.is_applied(14, 2)); + assert!( + scheduler + .owed + .contains_key(&(aborted, OwedKind::CompletionAck)) + ); + assert_eq!( + scheduler.pending.get(&committed).map(|p| p.commit_state), + Some(CommitState::Following), + "a committed write slice waits for its redo" + ); + assert!(!scheduler.applied.is_applied(15, 0)); + } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs index d6824170e..97d6acb33 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs @@ -12,22 +12,22 @@ use crate::control::cluster::calvin::scheduler::driver::core::halt::{ use crate::control::cluster::calvin::scheduler::driver::core::owed::SchedulerProposal; use crate::control::cluster::calvin::scheduler::driver::core::scheduler::Scheduler; use crate::control::cluster::calvin::scheduler::driver::core::staged_vote::{ - StagedVote, staged_commit_vote, + StageVoteError, StagedVote, staged_commit_vote, }; use crate::control::cluster::calvin::scheduler::driver::types::CommitState; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; impl Scheduler { - /// Cast this participant's local commit vote for a staged transaction, then - /// PARK it on the cross-shard commit barrier awaiting the durable GLOBAL - /// verdict — it does NOT self-decide flush-or-drop on its local vote. + /// Cast this leader's commit vote for a staged transaction, then PARK it + /// on the cross-shard commit barrier awaiting the durable GLOBAL verdict. + /// It never self-decides commit or abort on its local vote. /// /// The staged executor response is validate-only: its `stage_vote` is this /// shard's local commit vote, read by [`staged_commit_vote`]. A stage /// response the scheduler cannot read a vote from halts apply. It never - /// counts as a commit. The leader proposes that vote via the sequencer - /// Raft group. The sequencer aggregates all participants' votes into a - /// single authoritative `SequencerEntry::Verdict`, applied on every replica. + /// counts as a commit. The vote travels through the sequencer Raft group, + /// which aggregates every participant's vote into one authoritative + /// `SequencerEntry::Verdict`, applied on every replica. /// /// This method moves the txn to [`CommitState::AwaitingVerdict`] WITHOUT /// dispatching a resolve or drop, then immediately probes @@ -35,9 +35,7 @@ impl Scheduler { /// push we raced) it resumes at once via [`Self::resume_on_verdict`]; /// otherwise it stays parked, holding locks and its staged buffer, until the /// verdict push, a later probe, or the stall re-probe sweep delivers the - /// verdict. Resuming (in `resume_on_verdict`) is where the flush/drop is - /// dispatched and the flushed/dropped counters bump — using the GLOBAL - /// verdict, never the local vote. + /// verdict. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn resolve_staged_commit( &mut self, txn_id: TxnId, @@ -45,7 +43,7 @@ impl Scheduler { ) { // A staged error is always an abort vote. A response with no readable // vote halts: counting it as a commit will let a failed participant - // flush after its peers received a global commit verdict. + // commit after its peers received a global commit verdict. let superseded = self .pending .get(&txn_id) @@ -62,9 +60,16 @@ impl Scheduler { Ok(vote) => vote, Err(error) => { self.metrics.record_executor_error(); + let reason = match error { + StageVoteError::CoreFailStopped { .. } => HaltReason::CoreFailStopped, + StageVoteError::Missing { .. } + | StageVoteError::CommitWithoutSuccess { .. } => { + HaltReason::StageVoteInvalid + } + }; self.halt_apply( txn_id, - HaltReason::StageVoteInvalid, + reason, HaltStep::Stage, format!("{error}: {}", error_response_text("stage", staged_response)), ); @@ -73,26 +78,20 @@ impl Scheduler { } }; - // Durably propose this participant's commit vote via the sequencer - // Raft group, leader-guarded: only the data-group leader ran - // read-set validation, so only a leader's vote is - // authoritative. The sequencer aggregates every participant's vote into - // the single global verdict this txn parks on below. An abort travels as + // Durably propose this leader's commit vote via the sequencer Raft + // group. Only the leader stages, so its vote is the vShard's. The + // sequencer aggregates every participant's vote into the single + // global verdict this txn parks on below. An abort travels as // `AbortVote` so its cause survives to the coordinator. The vote stays // owed until the tally holds it, so a refused or dropped proposal is - // proposed again rather than lost. - // - // A follower owes an abort vote and proposes it only if it comes to - // lead before any vote of this vShard applies: the previous leader - // lost its leadership, or its scheduler cannot run. A follower ran - // none of the leader-only checks, so abort is the only vote it can - // cast. The txn fails with a retryable conflict and never hangs. - let abort = if self.is_group_leader() { - vote.abort_reason() - } else { - Some(nodedb_cluster::calvin::AbortReason::SerializationConflict) - }; - self.propose_sequencer_entry(txn_id, SchedulerProposal::Vote { abort }); + // proposed again rather than lost. A promoted leader's vote for a + // txn whose vote applied already changes nothing: the first counts. + self.propose_sequencer_entry( + txn_id, + SchedulerProposal::Vote { + abort: vote.abort_reason(), + }, + ); if vote == StagedVote::SerializationConflict { // The staged slice's read-set was no longer current: observe it on @@ -110,14 +109,14 @@ impl Scheduler { // this local vote, decides. If the txn already vanished (torn down // elsewhere), there is nothing to park. // - // A stage error parks too. A deterministic error fails on every - // replica, the leader votes abort, and every replica drops. A local - // error on a follower leaves the leader's commit vote standing: - // `resume_on_verdict` halts on that COMMIT verdict. + // A stage error parks too. The leader votes abort, and every + // participant drops at the abort verdict. A COMMIT verdict for it + // can follow only a vote an earlier leader cast: `resume_on_verdict` + // halts on it. match self.pending.get_mut(&txn_id) { Some(pending) => { pending.commit_state = CommitState::AwaitingVerdict; - // A superseded slice cannot flush here: its collection is gone. + // A superseded slice cannot commit here: its collection is gone. pending.stage_error = match vote { StagedVote::ParticipantError | StagedVote::PredictionDrift => { Some(error_response_text("stage", staged_response)) @@ -169,12 +168,14 @@ mod tests { use nodedb_cluster::calvin::{AbortReason, SequencerEntry}; use super::*; + use crate::bridge::envelope::ErrorCode; use crate::bridge::envelope::{StageVote, Status}; + use crate::control::cluster::calvin::scheduler::driver::core::owed::OwedKind; use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ - CapturingProposer, elect_data_group_leader, + CapturingProposer, lead_data_group, }; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ - build_test_scheduler, make_sequenced_txn, staged_pending, staged_response, + build_test_scheduler, error_response, make_sequenced_txn, staged_pending, staged_response, }; const VSHARD: u32 = 7; @@ -191,7 +192,7 @@ mod tests { let (mut scheduler, dir) = build_test_scheduler(VSHARD); let proposer = CapturingProposer::accepting(); scheduler.sequencer_proposer = proposer.clone(); - elect_data_group_leader(&scheduler); + lead_data_group(&mut scheduler); scheduler.pending.insert( txn_id, staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id), @@ -284,4 +285,61 @@ mod tests { Some(HaltReason::StageVoteInvalid) ); } + + /// A fail-stopped core refuses the stage and every later request. The + /// leader halts on it and votes nothing. + #[tokio::test] + async fn a_fail_stopped_core_halts_the_stage_and_never_votes() { + let txn_id = TxnId::new(33, 0); + let (mut scheduler, _dir, proposer) = staged_leader(txn_id); + + scheduler.resolve_staged_commit( + txn_id, + &error_response(ErrorCode::CoreFailStopped { + core_id: 0, + detail: "rollback failed".to_string(), + }), + ); + + assert_eq!(proposer.attempt_count(), 0, "no vote is proposed"); + assert_eq!( + scheduler.apply_halt().map(|h| h.reason), + Some(HaltReason::CoreFailStopped) + ); + assert!(!scheduler.applied.is_applied(33, 0)); + } + + /// The leader's vote is its real one, never a forced abort. A node that + /// lost its Raft leadership after it staged keeps the vote owed and + /// proposes nothing. + #[tokio::test] + async fn a_vote_is_proposed_only_while_this_node_leads() { + let txn_id = TxnId::new(34, 0); + let (mut scheduler, _dir) = build_test_scheduler(VSHARD); + let proposer = CapturingProposer::accepting(); + scheduler.sequencer_proposer = proposer.clone(); + scheduler.pending.insert( + txn_id, + staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id), + ); + + scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); + + assert_eq!(proposer.attempt_count(), 0, "a non-leader proposes nothing"); + assert!(scheduler.owed.contains_key(&(txn_id, OwedKind::Vote))); + lead_data_group(&mut scheduler); + scheduler.retry_owed_sequencer_entries(); + assert_eq!( + proposer.accepted(), + vec![SequencerEntry::Vote { + epoch: 34, + position: 0, + vshard: VSHARD, + }], + "the vote is the stage's commit, not a forced abort" + ); + } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs index 576e3fb9b..0f0049ec7 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs @@ -5,10 +5,9 @@ //! [`Scheduler::handle_completion`] is the `completion_rx` arm of the main //! `select!` loop: it classifies each executor response (disconnect, or the //! txn's commit-resolution state) and routes it to the matching handler. -//! Every txn threads through the commit-barrier states in -//! [`super::commit_resolve`]; a txn parked in -//! [`CommitState::AwaitingVerdict`] has no outstanding bridge, so a completion -//! for it is a no-op that keeps it parked. +//! Only the answer to the request a txn waits for drives it. An answer to a +//! step the txn left, after a demotion or an install from another copy, is +//! ignored. use super::super::types::CommitState; use super::halt::{HaltReason, HaltStep}; @@ -21,24 +20,28 @@ impl Scheduler { /// Process a completed executor response (or disconnected channel). /// /// Called from the `completion_rx` arm of the main `select!` loop. - pub(in crate::control::cluster::calvin::scheduler::driver::core) async fn handle_completion( + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn handle_completion( &mut self, txn_id: TxnId, request_id: RequestId, resp_opt: Option, ) { - let Some(commit_state) = self.pending.get(&txn_id).map(|p| p.commit_state) else { - // A response bridge exists only for a pending txn, and the txn - // leaves `pending` only after its last response. - tracing::error!( + if !self.awaits(txn_id, request_id) { + tracing::debug!( vshard_id = self.vshard_id, request_id = request_id.as_u64(), epoch = txn_id.epoch, position = txn_id.position, - "calvin: executor completion for a txn with no pending entry; ignoring" + "calvin: executor answer to a step the txn left; ignoring" ); return; + } + let Some(commit_state) = self.pending.get(&txn_id).map(|p| p.commit_state) else { + return; }; + if let Some(pending) = self.pending.get_mut(&txn_id) { + pending.awaiting = None; + } let response = match resp_opt { Some(r) => r, None => { @@ -66,10 +69,6 @@ impl Scheduler { .unwrap_or(0); self.metrics.record_executor_txn_duration_ms(elapsed_ms); - // A txn resolves through its commit-barrier state, OLLP answer - // included. The flush installs a resolved redo record and runs no - // OLLP check, so an OLLP answer under a later state is a failed step - // that halts and holds. It never settles as a retry. match commit_state { CommitState::Staged => { // Stage failures remain participants in the barrier: @@ -81,18 +80,15 @@ impl Scheduler { CommitState::AwaitingRedoResolve => { self.finish_redo_resolve(txn_id, response); } - CommitState::AwaitingResolve { - committed, - redo_lsn, - } => { - self.finish_resolved_commit(txn_id, response, committed, redo_lsn) - .await; + CommitState::AwaitingDrop => { + self.finish_drop(txn_id, &response); } - CommitState::AwaitingFlushTurn { .. } + CommitState::Following + | CommitState::AwaitingVerdict | CommitState::AwaitingResolveTurn - | CommitState::AwaitingVersionRecord => { - // A txn waiting for its flush or resolve turn has no request - // out. A write-version record drains its own response. + | CommitState::AwaitingRedoApply { .. } => { + // A txn in these states sends no request it waits for. A + // recorded request names a step it left. tracing::warn!( vshard_id = self.vshard_id, request_id = request_id.as_u64(), @@ -102,22 +98,6 @@ impl Scheduler { "calvin: executor completion for a txn with no request out; ignoring" ); } - CommitState::AwaitingVerdict => { - // A parked txn dispatched no executor request (it is waiting on - // the cross-shard verdict, not the Data Plane), so no response - // bridge is outstanding for it — a completion here indicates a - // logic error, not a real Data-Plane reply. Do NOT run the commit - // tail or release locks (that can tear the transaction); - // remain parked and let the verdict path resume it. - tracing::warn!( - vshard_id = self.vshard_id, - request_id = request_id.as_u64(), - epoch = txn_id.epoch, - position = txn_id.position, - "calvin: executor completion for a txn parked on the commit barrier; \ - ignoring (no bridge is outstanding while AwaitingVerdict)" - ); - } } } } @@ -136,20 +116,13 @@ mod tests { /// A response channel that closes before a response leaves the txn's /// outcome unknown: the txn stays pending and unapplied, the scheduler /// halts, and the node marker names the txn. - #[tokio::test] - async fn disconnected_response_holds_txn_unapplied_and_sets_node_marker() { + #[test] + fn disconnected_response_holds_txn_unapplied_and_sets_node_marker() { let txn_id = TxnId::new(5, 1); - let (mut scheduler, _dir) = scheduler_with_pending( - txn_id, - CommitState::AwaitingResolve { - committed: true, - redo_lsn: None, - }, - ); + let (mut scheduler, _dir) = + scheduler_with_pending(txn_id, CommitState::AwaitingRedoResolve); - scheduler - .handle_completion(txn_id, RequestId::new(9), None) - .await; + scheduler.handle_completion(txn_id, RequestId::new(9), None); assert!( !scheduler.applied.is_applied(5, 1), @@ -163,7 +136,7 @@ mod tests { let marker = scheduler.shared.sequencer_halt.apply_halt().report(); assert_eq!( marker.map(|h| (h.vshard_id, h.epoch, h.position, h.step)), - Some((7, 5, 1, "flush")) + Some((7, 5, 1, "resolve")) ); assert_eq!(scheduler.metrics.apply_halted.load(Ordering::Relaxed), 1); assert_eq!( @@ -173,21 +146,18 @@ mod tests { } /// A second disconnect keeps the first cause and holds its txn too. - #[tokio::test] - async fn second_halt_keeps_the_first_cause() { + #[test] + fn second_halt_keeps_the_first_cause() { let first = TxnId::new(5, 1); let second = TxnId::new(6, 0); let (mut scheduler, _dir) = scheduler_with_pending(first, CommitState::Staged); let mut pending = staged_pending(make_sequenced_txn(6, 0), second); pending.commit_state = CommitState::AwaitingRedoResolve; + pending.awaiting = Some(RequestId::new(10)); scheduler.pending.insert(second, pending); - scheduler - .handle_completion(first, RequestId::new(9), None) - .await; - scheduler - .handle_completion(second, RequestId::new(10), None) - .await; + scheduler.handle_completion(first, RequestId::new(9), None); + scheduler.handle_completion(second, RequestId::new(10), None); assert!(!scheduler.applied.is_applied(6, 0)); assert!(scheduler.pending.contains_key(&second)); @@ -198,30 +168,23 @@ mod tests { ); } - /// A committed flush that answers `OllpRetryRequired` wrote nothing on - /// this replica. The txn stays pending and unapplied, its redo records - /// stay open, and the scheduler halts on the flush. - #[tokio::test] - async fn an_ollp_answer_to_a_committed_flush_halts_and_holds() { + /// A refused resolve of a committed txn leaves no redo to propose. The + /// txn stays pending and unapplied, and the scheduler halts on the + /// resolve. + #[test] + fn a_refused_resolve_of_a_committed_txn_halts_and_holds() { let txn_id = TxnId::new(5, 1); - let (mut scheduler, _dir) = scheduler_with_pending( + let (mut scheduler, _dir) = + scheduler_with_pending(txn_id, CommitState::AwaitingRedoResolve); + + scheduler.handle_completion( txn_id, - CommitState::AwaitingResolve { - committed: true, - redo_lsn: None, - }, + RequestId::new(9), + Some(error_response( + crate::bridge::envelope::ErrorCode::OllpRetryRequired, + )), ); - scheduler - .handle_completion( - txn_id, - RequestId::new(9), - Some(error_response( - crate::bridge::envelope::ErrorCode::OllpRetryRequired, - )), - ) - .await; - assert!(!scheduler.applied.is_applied(5, 1)); assert!( scheduler.pending.contains_key(&txn_id), @@ -229,7 +192,28 @@ mod tests { ); assert_eq!( scheduler.apply_halt().map(|h| h.reason), - Some(HaltReason::FlushFailed) + Some(HaltReason::ResolveFailed) + ); + } + + /// An answer to a request the txn no longer waits for, as after a + /// demotion, changes nothing. + #[test] + fn an_answer_to_a_step_the_txn_left_is_ignored() { + let txn_id = TxnId::new(5, 1); + let (mut scheduler, _dir) = + scheduler_with_pending(txn_id, CommitState::AwaitingRedoResolve); + + scheduler.handle_completion(txn_id, RequestId::new(8), None); + + assert!(scheduler.apply_halt().is_none()); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingRedoResolve) + ); + assert_eq!( + scheduler.pending.get(&txn_id).and_then(|p| p.awaiting), + Some(RequestId::new(9)) ); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs index 00565b04e..92ec0d9d7 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs @@ -1,19 +1,16 @@ // SPDX-License-Identifier: BUSL-1.1 -//! A backup's cut marker on this vShard's scheduler, and the tenant write -//! mark a committed Calvin transaction records. +//! A backup's cut marker on this vShard's scheduler. //! //! The scheduler reports a marker to the node's cut registry once every -//! transaction delivered before it finished: installed by its flush, or -//! dropped under an abort verdict. The backup waiting on the marker then +//! transaction delivered before it finished: installed from its stamped +//! redo, or ended with no log entry. The backup waiting on the marker then //! snapshots a vShard that holds every one of them. Every transaction //! delivered after the marker records a commit HLC above the marker's -//! watermark, so a restore of that backup refuses it. +//! watermark, so a restore of that backup refuses it. The data-group apply +//! raises the tenant's write mark from the redo entry's commit HLC. use super::scheduler::Scheduler; -use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; -use crate::control::security::auth_fence::cluster::group_of_vshard; -use crate::control::state::tenant_marks::MarkSite; impl Scheduler { /// Receive a backup's cut marker carrying `hlc`, in input order. @@ -44,94 +41,4 @@ impl Scheduler { } self.cut_floors.fold(self.applied.fully_applied_epoch()); } - - /// The commit HLC of the transaction `txn_id`: the instant every WAL - /// record of its install carries. A transaction whose state is gone takes - /// the node's HLC now. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn txn_commit_hlc( - &self, - txn_id: TxnId, - ) -> u64 { - match self.pending.get(&txn_id) { - Some(pending) => self - .cut_floors - .commit_hlc(pending.txn.epoch, pending.txn.epoch_system_ms), - None => self.shared.hlc_clock.now().wall_ns, - } - } - - /// Record the commit HLC of the committed transaction `txn_id` on its - /// tenant's observed write high-water, before its `CompletionAck` lets - /// the coordinator acknowledge the COMMIT. - /// - /// Only a slice that carries a primary user data write records it. The - /// implicit edge cleanup that dual-homes alongside one writes no user row - /// of its own. - /// - /// A transaction that re-issues a RESTORE raises the tenant's restore - /// mark under its restore id instead, so a retry of that RESTORE finds - /// only its own marks. Its commit HLC still folds into this node's clock, - /// so a later backup here stamps a newer watermark. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn record_calvin_write_mark( - &self, - txn_id: TxnId, - ) { - let Some(pending) = self.pending.get(&txn_id) else { - return; - }; - if !pending.has_primary_write { - return; - } - let tx_class = &pending.txn.tx_class; - let tenant_id = tx_class.tenant_id.as_u64(); - let restore_id = tx_class.restore_id; - let collection = tx_class.write_set.0.first().map(|set| set.collection()); - let commit_hlc = self - .cut_floors - .commit_hlc(pending.txn.epoch, pending.txn.epoch_system_ms); - if restore_id == 0 { - self.shared - .advance_tenant_write_hlc(tenant_id, commit_hlc, "calvin flush", collection); - } else { - self.shared - .hlc_clock - .update(nodedb_types::Hlc::new(commit_hlc, 0)); - } - - // The durable mark, in the data group that homes this vShard, lands - // before the ack too: RESTORE's guard reads it on every node, after a - // restart as well. - let group_id = match group_of_vshard(&self.shared, self.vshard_id) { - Ok(group_id) => group_id, - Err(error) => { - tracing::error!( - vshard_id = self.vshard_id, - %error, - "calvin: no data group for this vShard; the commit's write mark stays \ - in memory only" - ); - return; - } - }; - let marks = &self.shared.tenant_marks; - if restore_id == 0 { - marks.raise( - group_id, - tenant_id, - commit_hlc, - MarkSite::CalvinFlush, - collection, - ); - } else { - marks.raise_restore(group_id, tenant_id, commit_hlc, collection, restore_id); - } - if let Err(error) = marks.persist(self.shared.credentials.catalog()) { - // The mark stays pending; the next persist on this node writes it. - tracing::error!( - vshard_id = self.vshard_id, - %error, - "calvin: the commit's write mark did not persist yet" - ); - } - } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs index 576b204ef..99779c575 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs @@ -10,15 +10,11 @@ //! refusal halts the scheduler (see [`super::halt`]). use std::collections::VecDeque; -use std::sync::atomic::Ordering; use std::time::Instant; -use nodedb_physical::physical_plan::PhysicalPlan; -use nodedb_physical::physical_plan::meta::MetaOp; - use super::halt::{HaltReason, HaltStep}; use super::scheduler::Scheduler; -use crate::bridge::dispatch::{DispatchRefusal, JournalGroup}; +use crate::bridge::dispatch::DispatchRefusal; use crate::bridge::envelope::Request; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; use crate::types::RequestId; @@ -32,12 +28,23 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) enum DispatchSt StageActive, /// `CalvinResolve` of a committed staged txn. Resolve, - /// `CalvinFlush` of a committed staged txn. - Flush, - /// `CalvinDrop` of an aborted staged txn. + /// `CalvinDrop` of a staged txn that ends with no log entry. The txn + /// completes on its answer. Drop, - /// One-way `RecordCalvinWriteVersions` of a committed txn. - WriteVersionRecord, + /// `CalvinDrop` of staged state the txn no longer needs: a demoted + /// leader's, or a restaged slice whose redo applied. Its answer is + /// drained and read by nothing. + Discard, +} + +impl DispatchStep { + /// Whether the txn waits for this step's answer. + fn is_awaited(self) -> bool { + match self { + Self::StageStatic | Self::StageActive | Self::Resolve | Self::Drop => true, + Self::Discard => false, + } + } } /// Result of [`Scheduler::dispatch_sequenced`]. @@ -57,8 +64,6 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) struct Deferred txn_id: TxnId, step: DispatchStep, request: Request, - /// The record group the request's write set journals into. - journal: Option, } /// FIFO of requests refused at capacity, in refusal order. @@ -77,6 +82,9 @@ impl Scheduler { /// Register `request` in the tracker, dispatch it, and cancel the /// registration on refusal. /// + /// An awaited step records the request on the txn's pending entry, so + /// only this request's answer drives the txn. + /// /// A capacity refusal appends the request to the deferred FIFO and /// returns [`DispatchOutcome::Deferred`]. The caller keeps the txn in /// flight. Any other refusal returns [`DispatchOutcome::Failed`], and the @@ -87,19 +95,12 @@ impl Scheduler { step: DispatchStep, request: Request, ) -> DispatchOutcome { - self.dispatch_sequenced_journalled(txn_id, step, request, None) - } - - /// [`Self::dispatch_sequenced`] for a request whose write set journals - /// into `journal`. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn dispatch_sequenced_journalled( - &mut self, - txn_id: TxnId, - step: DispatchStep, - request: Request, - journal: Option, - ) -> DispatchOutcome { - match self.send_once(txn_id, step, request, journal) { + if step.is_awaited() + && let Some(pending) = self.pending.get_mut(&txn_id) + { + pending.awaiting = Some(request.request_id); + } + match self.send_once(txn_id, step, request) { Attempt::Sent => DispatchOutcome::Sent, Attempt::Capacity(parked) => { self.deferred.push_back(*parked); @@ -126,14 +127,6 @@ impl Scheduler { self.has_deferred_dispatch() && !self.is_apply_halted() } - /// Whether a refused request of `txn_id` waits for capacity. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn has_parked_dispatch( - &self, - txn_id: TxnId, - ) -> bool { - self.deferred.iter().any(|parked| parked.txn_id == txn_id) - } - /// Number of refused requests waiting for capacity. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn deferred_dispatch_len( &self, @@ -146,6 +139,10 @@ impl Scheduler { /// Stops at the first capacity refusal, which goes back to the FIFO /// front. A terminal refusal runs the step's terminal handling, and stops /// the pass once the scheduler halted. + /// + /// A request of a txn that left `pending`, or that waits for another + /// request now, is dropped. A discard still goes out: it clears staged + /// state the txn left on its core. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn redispatch_deferred( &mut self, ) { @@ -154,25 +151,20 @@ impl Scheduler { txn_id, step, mut request, - journal, } = parked; - if !self.pending.contains_key(&txn_id) { - tracing::error!( + if step.is_awaited() && !self.awaits(txn_id, request.request_id) { + tracing::debug!( vshard_id = self.vshard_id, epoch = txn_id.epoch, position = txn_id.position, ?step, - "calvin: parked dispatch for a txn no longer pending; one step per txn is broken, discarding" + "calvin: parked dispatch of a step the txn left; discarding" ); continue; } - self.refresh_deferred_request(step, &mut request); - match self.send_once(txn_id, step, request, journal) { - Attempt::Sent => { - if step == DispatchStep::WriteVersionRecord { - self.complete_after_version_record(txn_id); - } - } + request.deadline = self.request_deadline(); + match self.send_once(txn_id, step, request) { + Attempt::Sent => {} Attempt::Capacity(parked) => { self.deferred.push_front(*parked); break; @@ -189,13 +181,23 @@ impl Scheduler { .set_dispatch_deferred_depth(self.deferred.len()); } + /// Whether `txn_id` is pending and waits for the answer to `request_id`. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn awaits( + &self, + txn_id: TxnId, + request_id: RequestId, + ) -> bool { + self.pending + .get(&txn_id) + .is_some_and(|pending| pending.awaiting == Some(request_id)) + } + /// Run the terminal handling of a dispatch the dispatcher refused for a /// reason other than capacity. /// /// Every refusal halts the scheduler: the txn keeps its `pending` entry /// and locks, and its position stays unapplied. A refusal during shutdown - /// holds the txn the same way. A lost write-version record halts too: - /// read-set validation reads those versions. + /// holds the txn the same way. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn fail_dispatch_step( &mut self, txn_id: TxnId, @@ -205,9 +207,7 @@ impl Scheduler { let halt_step = match step { DispatchStep::StageStatic | DispatchStep::StageActive => HaltStep::Stage, DispatchStep::Resolve => HaltStep::Resolve, - DispatchStep::Flush => HaltStep::Flush, - DispatchStep::Drop => HaltStep::Drop, - DispatchStep::WriteVersionRecord => HaltStep::WriteVersionRecord, + DispatchStep::Drop | DispatchStep::Discard => HaltStep::Drop, }; self.halt_apply( txn_id, @@ -218,38 +218,12 @@ impl Scheduler { } /// One send attempt: register, dispatch, and cancel on refusal. - fn send_once( - &mut self, - txn_id: TxnId, - step: DispatchStep, - request: Request, - journal: Option, - ) -> Attempt { - // A crash test holds one collection's flush here: the redo record is - // appended and no core holds the flush. The flush waits in the - // re-send queue, as at capacity, so the scheduler keeps running. - #[cfg(feature = "failpoints")] - if step == DispatchStep::Flush && crate::control::fail_gate::holds_flush(&request.plan) { - tracing::info!( - vshard_id = self.vshard_id, - epoch = txn_id.epoch, - position = txn_id.position, - "calvin: flush held at a fail point" - ); - return Attempt::Capacity(Box::new(DeferredDispatch { - txn_id, - step, - request, - journal, - })); - } + fn send_once(&mut self, txn_id: TxnId, step: DispatchStep, request: Request) -> Attempt { let request_id = request.request_id; let resp_rx = self.shared.tracker.register(request_id); let result = match self.shared.dispatcher.lock() { - Ok(mut dispatcher) => dispatcher.try_dispatch_journalled(request, journal.clone()), - Err(poisoned) => poisoned - .into_inner() - .try_dispatch_journalled(request, journal.clone()), + Ok(mut dispatcher) => dispatcher.try_dispatch(request), + Err(poisoned) => poisoned.into_inner().try_dispatch(request), }; let refusal = match result { Ok(()) => { @@ -275,7 +249,6 @@ impl Scheduler { txn_id, step, request, - journal, })) } other => Attempt::Failed(other), @@ -299,46 +272,18 @@ impl Scheduler { } self.spawn_response_bridge(txn_id, request_id, resp_rx); } - DispatchStep::Resolve | DispatchStep::Flush | DispatchStep::Drop => { + DispatchStep::Resolve | DispatchStep::Drop => { self.spawn_response_bridge(txn_id, request_id, resp_rx); } - DispatchStep::WriteVersionRecord => { - // One-way record: drain the response so it routes to a live - // receiver, then discard it. + DispatchStep::Discard => { + // Drain the answer so it routes to a live receiver. tokio::spawn(async move { let mut rx = resp_rx; let _ = rx.recv().await; }); - self.shared - .calvin - .counters - .write_versions_recorded - .fetch_add(1, Ordering::Relaxed); } } } - - /// Refresh the dispatch-time fields of a parked request before a re-send. - /// - /// The deadline restarts from now. A stage request re-reads group - /// leadership, which the Data Plane uses to gate OLLP verification. - fn refresh_deferred_request(&self, step: DispatchStep, request: &mut Request) { - request.deadline = self.request_deadline(); - if !matches!(step, DispatchStep::StageStatic | DispatchStep::StageActive) { - return; - } - if let PhysicalPlan::Meta( - MetaOp::CalvinExecuteStatic { - is_group_leader, .. - } - | MetaOp::CalvinExecuteActive { - is_group_leader, .. - }, - ) = &mut request.plan - { - *is_group_leader = self.is_group_leader(); - } - } } #[cfg(test)] @@ -352,6 +297,7 @@ mod tests { use super::*; use crate::control::cluster::calvin::scheduler::driver::core::halt::HaltReason; use crate::control::cluster::calvin::scheduler::driver::core::intake::IntakeClosure; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::lead_data_group; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ begin_data_plane_drain, build_test_scheduler_with_data_side, fill_tenant_inflight, make_validate_only_txn, test_coll_vshard, @@ -365,6 +311,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, _data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); begin_data_plane_drain(&scheduler.shared); let txn_id = TxnId::new(3, 0); @@ -403,6 +350,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); let shared = Arc::clone(&scheduler.shared); fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); let txn_id = TxnId::new(3, 0); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs index 54caa0015..08405f6cd 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Active (dependent-read) transaction dispatch: submits a -//! `CalvinExecuteActive` task once all passive results have landed. +//! Active (dependent-read) transaction dispatch on the data-group leader: +//! submits a `CalvinExecuteActive` task once all passive results have landed. use std::time::Instant; @@ -12,7 +12,8 @@ use nodedb_physical::physical_plan::meta::MetaOp; use super::super::deferred::{DispatchOutcome, DispatchStep}; use super::super::scheduler::Scheduler; use super::primary_write::{ - plans_have_primary_write, plans_have_returning, txn_has_non_derived_write, + plans_have_primary_write, plans_have_returning, plans_raise_write_mark, + txn_has_non_derived_write, }; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; @@ -71,12 +72,16 @@ impl Scheduler { return; } }; - if !self.bind_local_identities(&mut plans, txn.tx_class.database_id, tenant_id, txn_id) { + let Some(identities) = + self.bind_local_identities(&mut plans, txn.tx_class.database_id, tenant_id, txn_id) + else { return; - } + }; let has_primary_write = plans_have_primary_write(&plans, has_non_derived_write); + let raises_write_mark = plans_raise_write_mark(&plans, has_primary_write); let has_returning = plans_have_returning(&plans); - let flush_scope = super::super::super::types::FlushScope::of_plans(&plans); + let mut scope = super::super::super::types::SliceScope::of_plans(&plans); + scope.identities = identities; let plan = PhysicalPlan::Meta(MetaOp::CalvinExecuteActive { epoch, position, @@ -84,18 +89,15 @@ impl Scheduler { plans, injected_reads, epoch_system_ms: txn.epoch_system_ms, - is_group_leader: self.is_group_leader(), }); - // Calvin allocates the CalvinApplied WAL LSN post-apply (in the - // scheduler's response handler), so no committed LSN is known at - // dispatch time to stamp here. + // A stage writes no WAL record, so no committed LSN rides on it. let database_id = txn.tx_class.database_id; - let request = self.build_exempt_request(request_id, tenant_id, database_id, plan, None); + let request = self.build_exempt_request(request_id, tenant_id, database_id, plan); // The txn enters `pending` before the dispatch, so a stage refused at // capacity stays in flight with its locks until the re-send. - // Every replica checks the transaction's collection incarnations. + // The leader checks the transaction's collection incarnations. let (superseded, gates) = self.check_incarnations(&txn.tx_class); self.pending.insert( txn_id, @@ -105,26 +107,25 @@ impl Scheduler { // no-determinism: dispatch_time is scheduler observability, not Calvin WAL data dispatch_time: Instant::now(), has_primary_write, + raises_write_mark, has_returning, - // The commit's resolved redo fills them. - change_sets: Vec::new(), - // The dependent-read active path STAGES (leader-verify OLLP + + // The dependent-read active path STAGES (OLLP verify + // buffer, no base apply); its response drives the same - // resolve → redo → flush as the static path, for - // WAL-only-restart durability. `resolve_staged_commit` reads the - // `stage_vote` the active handler sets. + // resolve and redo proposal as the static path. + // `resolve_staged_commit` reads the `stage_vote` the active + // handler sets. commit_state: super::super::super::types::CommitState::Staged, + // The dispatch below records its request. + awaiting: None, // Set only once the txn parks in `AwaitingVerdict`. verdict_deadline: None, stage_error: None, - // Set once a committed txn appends its redo record. - redo_records: None, - flush_scope, + scope, + // Set once a committed slice resolves its redo. + redo: None, superseded, gates, ungated: false, - // Taken when the flush dispatches. - install_permit: None, }, ); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/bind_identities.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/bind_identities.rs index 9481dc9f2..d3cbed167 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/bind_identities.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/bind_identities.rs @@ -3,40 +3,43 @@ //! Install the pk → surrogate identities a local Calvin write slice carries. //! //! The coordinator assigned each surrogate at plan time in its own catalog. -//! Every participant — group leader and follower alike — applies the slice -//! to its Data Plane, so every participant installs the same binding here, -//! or a later point read by primary key resolves nothing on this node. +//! The data-group leader binds them here before it stages the slice, and +//! carries them in the slice's redo entry. Every other replica binds the +//! same identities before the redo installs, so a later point read by +//! primary key resolves the row on every node. use nodedb_physical::physical_plan::PhysicalPlan; use super::super::halt::{HaltReason, HaltStep}; use super::super::scheduler::Scheduler; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; -use crate::control::surrogate::bind_plan_identities; +use crate::control::surrogate::{CarriedIdentity, bind_plan_identities, collect_plan_identities}; use crate::types::{DatabaseId, TenantId}; impl Scheduler { - /// Bind every identity in `plans` first-wins and rewrite each surrogate + /// Bind every identity in `plans` first-wins, rewrite each surrogate /// slot with the authoritative value, the same walk the replicated-write - /// decoder runs. + /// decoder runs, and return the identities the slice's redo carries. /// - /// A catalog error is local to this replica, and its peers apply the - /// slice. So the scheduler halts: the txn is not yet in `pending`, its - /// locks stay held under its lock owner, and its position stays - /// unapplied. Returns `false` after the halt; the caller returns at once. + /// A catalog error is local to this node, and its peers apply the slice. + /// So the scheduler halts: the txn is not yet in `pending`, its locks + /// stay held under its lock owner, and its position stays unapplied. + /// Returns `None` after the halt; the caller returns at once. pub(super) fn bind_local_identities( &mut self, plans: &mut [PhysicalPlan], database_id: DatabaseId, tenant_id: TenantId, txn_id: TxnId, - ) -> bool { + ) -> Option> { let assigner = &self.shared.surrogate_assigner; - let bound = plans + let rewritten = plans .iter_mut() .try_for_each(|plan| bind_plan_identities(assigner, database_id, tenant_id, plan)); + let bound = rewritten + .and_then(|()| collect_plan_identities(assigner, database_id, tenant_id, plans)); match bound { - Ok(()) => true, + Ok(identities) => Some(identities), Err(e) => { self.halt_apply( txn_id, @@ -44,7 +47,7 @@ impl Scheduler { HaltStep::IdentityBind, format!("surrogate binding failed: {e}"), ); - false + None } } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/follow.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/follow.rs new file mode 100644 index 000000000..83a3790b9 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/follow.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A granted txn on a replica that does not lead the vShard's data group. +//! +//! The follower holds the txn's locks and stages nothing. It owes no vote. +//! The txn completes on an abort verdict, on a COMMIT verdict for a slice +//! with no local write, or once the slice's stamped redo applies from the +//! log. A promotion stages it. + +use std::time::Instant; + +use nodedb_cluster::calvin::types::SequencedTxn; + +use super::super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::driver::types::{ + CommitState, PendingTxn, SliceScope, +}; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + +impl Scheduler { + /// Hold `txn` `Following`. A verdict stored already resolves it at once. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn follow( + &mut self, + txn: SequencedTxn, + txn_id: TxnId, + lock_owner: TxnId, + ) { + let scope = self.follow_scope(&txn, txn_id); + self.pending.insert( + txn_id, + PendingTxn { + txn, + lock_owner, + // no-determinism: dispatch time is observability only. + dispatch_time: Instant::now(), + has_primary_write: false, + raises_write_mark: false, + has_returning: false, + commit_state: CommitState::Following, + awaiting: None, + verdict_deadline: None, + stage_error: None, + scope, + redo: None, + superseded: false, + gates: Vec::new(), + ungated: false, + }, + ); + if let Some(verdict) = self.registry.verdict(nodedb_cluster::calvin::TxnId::new( + txn_id.epoch, + txn_id.position, + )) { + self.resume_on_verdict(txn_id, verdict); + } + } + + /// The scope of `txn`'s slice on this vShard. Plans that do not decode + /// or route make the leader vote abort, so they hold no write here. + fn follow_scope(&self, txn: &SequencedTxn, txn_id: TxnId) -> SliceScope { + let local = + super::super::super::helpers::decode_plans(&txn.tx_class.plans).and_then(|plans| { + self.local_calvin_plans( + plans, + txn.tx_class.database_id, + txn_id.epoch, + txn_id.position, + ) + }); + match local { + Ok(local) => SliceScope::of_plans(&local), + Err(error) => { + tracing::debug!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + %error, + "calvin: follower holds a txn whose plans the leader rejects" + ); + SliceScope::of_plans(&[]) + } + } + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs index 170867c3d..42968d41d 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs @@ -1,18 +1,18 @@ // SPDX-License-Identifier: BUSL-1.1 -//! The incarnation check a Calvin transaction passes when a replica stages it. +//! The incarnation check a Calvin transaction passes when the leader stages +//! it. //! //! The coordinator stamps the incarnation of every collection the transaction -//! names. Each replica checks all of them, not only its own slice's, against +//! names. The leader checks all of them, not only its own slice's, against //! its committed catalog under the collections' gates, held shared until the //! transaction leaves `pending`. A mismatch makes the leader's vote an abort, //! and the sequencer's global verdict then aborts every participant alike: no -//! part of the transaction applies. A follower with a mismatch under a COMMIT -//! verdict halts rather than flush into a recreated collection. +//! part of the transaction applies. //! -//! A halt releases every gate no flush in flight needs, so a purge never waits -//! on a halted scheduler. A txn whose gates a halt released checks again, and -//! takes them back, before its flush. +//! A halt releases every gate, so a purge never waits on a halted +//! scheduler. A txn whose gates a halt released checks again, and takes them +//! back, before it proposes its redo. use std::collections::BTreeSet; @@ -20,7 +20,6 @@ use nodedb_cluster::calvin::types::TxClass; use nodedb_types::{CollectionKey, Hlc}; use tracing::error; -use super::super::super::types::CommitState; use super::super::scheduler::Scheduler; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; use crate::control::write_gate::{self, GateKey, SharedGate}; @@ -85,40 +84,26 @@ impl Scheduler { (superseded, gates) } - /// Release the gates a halt leaves nothing to protect. + /// Release every gate the pending txns hold, so a purge never waits on + /// this halted scheduler. /// - /// A halted scheduler never re-sends a parked request, and `stuck` never - /// applies before restart. Only a committed flush the Data Plane already - /// holds can still write, and it keeps its gates until it answers. Every - /// other txn gives its gates back, so a purge never waits on this - /// scheduler. + /// No write of a txn is in the Data Plane's hands under these gates: a + /// committed slice installs from the data-group log, and the apply loop + /// checks the incarnations its entry carries. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn release_gates_on_halt( &mut self, - stuck: TxnId, ) { - let release: Vec = self - .pending - .iter() - .filter(|(txn_id, pending)| { - **txn_id == stuck - || !writes_in_flight(pending.commit_state) - || self.has_parked_dispatch(**txn_id) - }) - .map(|(txn_id, _)| *txn_id) - .collect(); - for txn_id in release { - if let Some(pending) = self.pending.get_mut(&txn_id) - && !pending.ungated - { + for pending in self.pending.values_mut() { + if !pending.ungated { pending.gates.clear(); pending.ungated = true; } } } - /// Take back the gates of `txn_id` before its flush, when a halt released - /// them. Fails when a collection the txn names no longer holds its planned - /// incarnation: the flush cannot apply here. + /// Take back the gates of `txn_id` before it proposes its redo, when a + /// halt released them. Fails when a collection the txn names no longer + /// holds its planned incarnation: the redo cannot commit into it. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn regate( &mut self, txn_id: TxnId, @@ -149,18 +134,6 @@ impl Scheduler { } } -/// Whether a txn in `state` can have a write in the Data Plane's hands: a -/// committed flush. -fn writes_in_flight(state: CommitState) -> bool { - matches!( - state, - CommitState::AwaitingResolve { - committed: true, - .. - } - ) -} - #[cfg(test)] mod tests { use nodedb_cluster::calvin::types::CalvinIncarnation; @@ -185,8 +158,8 @@ mod tests { /// A transaction staged against `orders` before a purge and a same-name /// create is superseded on this replica, though `ledger` is unchanged. The /// leader's vote is an abort, which the global verdict applies to every - /// participant, `ledger`'s included. A replica that staged it under a - /// COMMIT verdict halts instead of flushing. + /// participant, `ledger`'s included. A leader that staged it under a + /// COMMIT verdict halts instead of committing it. #[test] fn a_txn_staged_before_a_recreate_is_superseded_everywhere() { let txn_id = TxnId::new(3, 0); @@ -237,7 +210,7 @@ mod tests { ); // The staged slice votes abort, whatever its own read-set says, and - // cannot flush under a COMMIT verdict. + // cannot commit under a COMMIT verdict. if let Some(pending) = scheduler.pending.get_mut(&txn_id) { pending.superseded = true; } @@ -270,11 +243,11 @@ mod tests { drop(reclaim); } - /// A replica that halts on a txn it never staged gives back the gates the + /// A leader that halts on a txn it never staged gives back the gates the /// txn held: a purge of its collection proceeds. The txn checks again - /// before a flush and fails once the collection was recreated. + /// before a proposal and fails once the collection was recreated. #[tokio::test] - async fn a_halt_releases_the_gates_and_a_later_flush_checks_again() { + async fn a_halt_releases_the_gates_and_a_later_proposal_checks_again() { let txn_id = TxnId::new(5, 0); let (mut scheduler, _dir) = scheduler_with_pending(txn_id, CommitState::AwaitingVerdict); let shared = std::sync::Arc::clone(&scheduler.shared); @@ -324,7 +297,7 @@ mod tests { assert_eq!(pending.gates.len(), 1, "the check took the gate back"); assert!(!pending.ungated); - scheduler.release_gates_on_halt(txn_id); + scheduler.release_gates_on_halt(); catalog .delete_collection(DatabaseId::DEFAULT, 1, "orders") .expect("purge"); @@ -336,7 +309,7 @@ mod tests { .expect("recreate"); assert!( scheduler.regate(txn_id).is_err(), - "a flush never lands in the recreated collection" + "a redo never commits into the recreated collection" ); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/mod.rs index f71c8d1c1..1042783f6 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/mod.rs @@ -1,9 +1,11 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Static and active (dependent-read) txn dispatch to the Data Plane. +//! Granted txn dispatch: the leader stages static and active (dependent-read) +//! txns on the Data Plane, and a follower holds them. mod active_dispatch; mod bind_identities; +mod follow; mod incarnation; mod primary_write; mod static_dispatch; diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/primary_write.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/primary_write.rs index 7e9cfca34..7211821b0 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/primary_write.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/primary_write.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Per-slice write classification: primary-write and RETURNING predicates -//! shared by the static and active dispatch paths. +//! Per-slice write classification: primary-write, write-mark and RETURNING +//! predicates shared by the static and active dispatch paths. use nodedb_physical::physical_plan::PhysicalPlan; @@ -21,7 +21,7 @@ pub(super) fn txn_has_non_derived_write(plans: &[PhysicalPlan]) -> bool { /// ONE deposited response, primary-write participants coalesce first-wins, and /// a derived participant's response describes a row the user's statement never /// named. A balance write that won that race handed an `INSERT` tag a count — -/// or, when its flush found the commit already resolved and answered with an +/// or, when its install found the commit already resolved and answered with an /// empty payload, no count at all — belonging to a different write entirely. /// /// `is_derived_side_effect` is the named predicate rather than an inline @@ -47,6 +47,17 @@ pub(super) fn plans_have_primary_write( .any(crate::control::planner::calvin::is_write_plan) } +/// Whether this vShard's slice raises its tenant's write mark when it +/// installs: a primary write that changes a tenant row. A slice that only +/// installs schema, such as a constraint set the leader's write gate +/// sequenced, changes no row, so a restore overwrites no write of it. +pub(super) fn plans_raise_write_mark(plans: &[PhysicalPlan], has_primary_write: bool) -> bool { + has_primary_write + && plans + .iter() + .any(crate::control::server::shared::write_admission::plan_writes_user_data) +} + /// Whether this vShard's slice carries a RETURNING-bearing write — a plan whose /// applied response is DATA-ROWs rather than a bare affected-count. Uses the /// SAME `describe_plan` classification the coordinator's response-shaping uses, @@ -57,3 +68,38 @@ pub(super) fn plans_have_returning(plans: &[PhysicalPlan]) -> bool { .iter() .any(|plan| matches!(describe_plan(plan), PlanKind::ReturningRows)) } + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_physical::physical_plan::{CrdtOp, KvOp}; + use nodedb_types::{DatabaseId, QualifiedCollection, Surrogate}; + + fn collection() -> QualifiedCollection { + QualifiedCollection::new(DatabaseId::DEFAULT, "c") + } + + /// A routed constraint set is the slice's primary write, and it changes + /// no row. Its install raises no write mark. + #[test] + fn only_a_slice_that_changes_a_row_raises_the_write_mark() { + let set = PhysicalPlan::Crdt(CrdtOp::SetConstraints { + collection: collection(), + constraint_version: 1, + constraints: Vec::new(), + }); + let put = PhysicalPlan::Kv(KvOp::Put { + collection: collection(), + key: b"k".to_vec(), + value: b"v".to_vec(), + ttl_ms: 0, + surrogate: Surrogate::new(1), + returning: None, + rls_filters: Vec::new(), + provenance: None, + }); + assert!(!plans_raise_write_mark(std::slice::from_ref(&set), true)); + assert!(plans_raise_write_mark(std::slice::from_ref(&put), true)); + assert!(!plans_raise_write_mark(&[put], false)); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/static_dispatch.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/static_dispatch.rs index 42b1a2b2b..b9c8b56e9 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/static_dispatch.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/static_dispatch.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Static-set ready-transaction dispatch: local-plan routing, group-leader -//! resolution, and the `CalvinExecuteStatic` submit/stage orchestration. +//! Static-set ready-transaction dispatch on the data-group leader: local-plan +//! routing and the `CalvinExecuteStatic` submit/stage orchestration. use std::time::Instant; @@ -13,7 +13,8 @@ use super::super::deferred::{DispatchOutcome, DispatchStep}; use super::super::routing::PlanRouting; use super::super::scheduler::Scheduler; use super::primary_write::{ - plans_have_primary_write, plans_have_returning, txn_has_non_derived_write, + plans_have_primary_write, plans_have_returning, plans_raise_write_mark, + txn_has_non_derived_write, }; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; use crate::types::DatabaseId; @@ -25,31 +26,11 @@ struct LocalSlice { body_plans: Vec, /// Every local plan is a body's, and a client plan homes elsewhere. body_only: bool, + /// The identities the local plans carry, bound in this node's catalog. + identities: Vec, } impl Scheduler { - /// Whether THIS node is currently the leader of the data-group owning this - /// scheduler's vshard. - /// - /// Stamped into the `CalvinExecute{Static,Active}` MetaOp at dispatch time - /// so the Data Plane runs the OLLP optimistic-lock verification (and emits - /// `OllpRetryRequired`) ONLY on the leader, while every replica applies the - /// carried predicted write-set verbatim — preserving Calvin determinism. - /// - /// Resolved via the existing routing → group-role check (no new election). - /// On a poisoned lock the inner guard is recovered; a momentarily-unknown - /// leadership (e.g. mid-election) resolves to `false`, i.e. follower-style - /// apply against the predicted set, which is always determinism-safe. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn is_group_leader( - &self, - ) -> bool { - let mr = match self.multi_raft.lock() { - Ok(g) => g, - Err(poisoned) => poisoned.into_inner(), - }; - mr.vshard_role_is_leader(self.vshard_id) - } - /// Filter a transaction's write plans down to the slice that homes to this /// scheduler's vShard. /// @@ -149,9 +130,11 @@ impl Scheduler { return; } }; - if !self.bind_local_identities(&mut local, txn.tx_class.database_id, tenant_id, txn_id) { + let Some(identities) = + self.bind_local_identities(&mut local, txn.tx_class.database_id, tenant_id, txn_id) + else { return; - } + }; // A participant with no local WRITE slice is either a READ-ONLY // participant — writes home elsewhere, but a read homes HERE, so it must @@ -191,6 +174,7 @@ impl Scheduler { plans: local, body_plans: local_body_plans, body_only, + identities, }, has_non_derived_write, ); @@ -252,6 +236,7 @@ impl Scheduler { plans, body_plans, body_only, + identities, } = slice; // The apply-slot identity (used in the CalvinExecuteStatic task and // error logs) is exactly `txn_id`; deriving it here keeps the two in @@ -260,9 +245,11 @@ impl Scheduler { let position = txn_id.position; let request_id = self.next_request_id(); let has_primary_write = plans_have_primary_write(&plans, has_non_derived_write); + let raises_write_mark = plans_raise_write_mark(&plans, has_primary_write); let has_returning = plans_have_returning(&plans); - let mut flush_scope = super::super::super::types::FlushScope::of_plans(&plans); - flush_scope.body_only = body_only; + let mut scope = super::super::super::types::SliceScope::of_plans(&plans); + scope.body_only = body_only; + scope.identities = identities; let database_id = txn.tx_class.database_id; let plan = PhysicalPlan::Meta(MetaOp::CalvinExecuteStatic { epoch, @@ -270,7 +257,6 @@ impl Scheduler { tenant_id, plans, epoch_system_ms: txn.epoch_system_ms, - is_group_leader: self.is_group_leader(), // The replicated read-set travels to the apply core so each // participant can check, at apply, whether its slice of the reads was // still current. Empty for pure-write / autocommit transactions. @@ -278,15 +264,12 @@ impl Scheduler { body_plans, }); - // Calvin allocates the CalvinApplied WAL LSN post-apply (in the - // scheduler's response handler), so no committed LSN is known at - // dispatch time to stamp here. - let mut request = self.build_exempt_request(request_id, tenant_id, database_id, plan, None); + // A stage writes no WAL record, so no committed LSN rides on it. + let mut request = self.build_exempt_request(request_id, tenant_id, database_id, plan); // The stage tags the rows it stages by the slice's source. - request.event_source = - super::super::request::slice_event_source(&txn.tx_class, &flush_scope); + request.event_source = super::super::request::slice_event_source(&txn.tx_class, &scope); - // Every replica checks the transaction's collection incarnations. + // The leader checks the transaction's collection incarnations. let (superseded, gates) = self.check_incarnations(&txn.tx_class); self.pending.insert( txn_id, @@ -296,24 +279,22 @@ impl Scheduler { // no-determinism: dispatch_time is scheduler observability, not Calvin WAL data dispatch_time: Instant::now(), has_primary_write, + raises_write_mark, has_returning, - // The commit's resolved redo fills them. - change_sets: Vec::new(), // This dispatch STAGES the txn (validate + buffer, no apply); - // its response carries the local commit vote that drives the - // subsequent flush-or-drop. + // its response carries the local commit vote. commit_state: super::super::super::types::CommitState::Staged, + // The dispatch below records its request. + awaiting: None, // Set only once the txn parks in `AwaitingVerdict`. verdict_deadline: None, stage_error: None, - // Set once a committed txn appends its redo record. - redo_records: None, - flush_scope, + scope, + // Set once a committed slice resolves its redo. + redo: None, superseded, gates, ungated: false, - // Taken when the flush dispatches. - install_permit: None, }, ); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs index 26a551366..57ea07a1f 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs @@ -1,12 +1,13 @@ // SPDX-License-Identifier: BUSL-1.1 -//! A transaction this vShard cannot stage: its plans are rejected, or its +//! A transaction the leader cannot stage: its plans are rejected, or its //! dependent reads never arrived. //! //! The txn still takes the commit barrier. It enters `pending`, holds its //! locks, and owes an abort vote. At the abort verdict it drops like any //! staged txn: it proposes its `CompletionAck`, releases its locks, and only -//! then marks its position applied. +//! then marks its position applied. A follower rejects the same plans and +//! holds the txn `Following` for the same verdict. use std::time::Instant; @@ -18,7 +19,7 @@ use nodedb_cluster::calvin::types::SequencedTxn; use super::super::owed::SchedulerProposal; use super::super::scheduler::Scheduler; use crate::control::cluster::calvin::scheduler::driver::types::{ - CommitState, FlushScope, PendingTxn, + CommitState, PendingTxn, SliceScope, }; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; @@ -26,8 +27,9 @@ impl Scheduler { /// Park `txn` with an abort vote for plans this vShard rejects. /// /// Plan decode, part assembly, and plan routing read only replicated - /// input. So every replica rejects the same plans, and every replica - /// votes `PlanRejected`. + /// input. So every replica rejects the same plans, and the leader votes + /// `PlanRejected`. A follower owes no vote: it holds the txn for the + /// abort verdict. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn reject_plan( &mut self, txn: SequencedTxn, @@ -35,6 +37,10 @@ impl Scheduler { lock_owner: TxnId, error: &crate::Error, ) { + if !self.role.is_leader() { + self.follow(txn, txn_id, lock_owner); + return; + } error!( vshard_id = self.vshard_id, epoch = txn_id.epoch, @@ -55,8 +61,8 @@ impl Scheduler { /// with an abort vote for `reason`. /// /// `stage_error` stays on the pending entry. A COMMIT verdict for the - /// txn then halts the scheduler, because this replica holds nothing to - /// flush. + /// txn then halts the scheduler, because this leader holds nothing to + /// resolve. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn park_unstaged( &mut self, txn: SequencedTxn, @@ -74,17 +80,17 @@ impl Scheduler { lock_owner, dispatch_time: now, has_primary_write: false, + raises_write_mark: false, has_returning: false, - change_sets: Vec::new(), commit_state: CommitState::AwaitingVerdict, + awaiting: None, verdict_deadline: Some(now + self.config.verdict_stall_warn()), stage_error: Some(stage_error), - redo_records: None, - flush_scope: FlushScope::default(), + scope: SliceScope::of_plans(&[]), + redo: None, superseded: false, gates: Vec::new(), ungated: false, - install_permit: None, }, ); self.propose_sequencer_entry( @@ -117,7 +123,7 @@ mod tests { use crate::bridge::dispatch::CoreChannelDataSide; use crate::bridge::envelope::{StageVote, Status}; use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ - CapturingProposer, elect_data_group_leader, + CapturingProposer, lead_data_group, }; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ build_test_scheduler_with_data_side, make_sequenced_txn, staged_pending, staged_response, @@ -141,7 +147,7 @@ mod tests { build_test_scheduler_with_data_side(vshard, Arc::clone(registry)); let proposer = CapturingProposer::accepting(); scheduler.sequencer_proposer = proposer.clone(); - elect_data_group_leader(&scheduler); + lead_data_group(&mut scheduler); (scheduler, dir, data_side, proposer) } @@ -239,16 +245,11 @@ mod tests { scheduler.resume_on_verdict(txn_id, false); assert_eq!( scheduler.pending.get(&txn_id).map(|p| p.commit_state), - Some(CommitState::AwaitingResolve { - committed: false, - redo_lsn: None - }), + Some(CommitState::AwaitingDrop), "vShard {vshard} leaves AwaitingVerdict for its drop" ); assert!(!scheduler.applied.is_applied(30, 0)); - scheduler - .finish_resolved_commit(txn_id, staged_response(Status::Ok, None), false, None) - .await; + scheduler.finish_drop(txn_id, &staged_response(Status::Ok, None)); assert!(!scheduler.pending.contains_key(&txn_id)); assert!(scheduler.applied.is_applied(30, 0)); registry.note_completion_ack(cluster_txn, vshard); @@ -279,10 +280,7 @@ mod tests { scheduler.dispatch_txn(undecodable_txn(32, 0), txn_id, txn_id); assert_eq!( scheduler.pending.get(&txn_id).map(|p| p.commit_state), - Some(CommitState::AwaitingResolve { - committed: false, - redo_lsn: None - }) + Some(CommitState::AwaitingDrop) ); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/drop_dispatch.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/drop_dispatch.rs new file mode 100644 index 000000000..c0d1b8c44 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/drop_dispatch.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Dispatch of a `CalvinDrop`: discard what a txn staged on its core. + +use nodedb_physical::physical_plan::PhysicalPlan; +use nodedb_physical::physical_plan::meta::MetaOp; + +use super::commit_redo::missing_pending_error; +use super::deferred::{DispatchOutcome, DispatchStep}; +use super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + +impl Scheduler { + /// Dispatch the drop of `txn_id`'s staged state as `step`: a + /// [`DispatchStep::Drop`] the txn completes on, or a + /// [`DispatchStep::Discard`] nothing waits for. + /// + /// A drop writes nothing, so no WAL LSN rides on it. A capacity refusal + /// returns [`DispatchOutcome::Deferred`]: the drop is parked for re-send + /// and the txn stays in flight. A txn with no `pending` entry returns + /// [`DispatchOutcome::Failed`]. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn dispatch_drop( + &mut self, + txn_id: TxnId, + step: DispatchStep, + ) -> DispatchOutcome { + let Some(pending) = self.pending.get(&txn_id) else { + return DispatchOutcome::Failed(missing_pending_error(txn_id)); + }; + let tenant_id = pending.txn.tx_class.tenant_id; + let database_id = pending.txn.tx_class.database_id; + let plan = PhysicalPlan::Meta(MetaOp::CalvinDrop { + epoch: txn_id.epoch, + position: txn_id.position, + }); + let request_id = self.next_request_id(); + let request = self.build_exempt_request(request_id, tenant_id, database_id, plan); + self.dispatch_sequenced(txn_id, step, request) + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/flush_turn.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/flush_turn.rs deleted file mode 100644 index dffa37acd..000000000 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/flush_turn.rs +++ /dev/null @@ -1,225 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Sequencer-order flushes. -//! -//! A vShard's transactions stage, vote, and resolve in any order: their -//! responses and verdicts arrive in any order. Their flushes run one at a -//! time, in `(epoch, position)` order. A committed transaction's flush -//! dispatches once no lower transaction of this vShard is unfinished. -//! -//! One core applies every request of a vShard, in dispatch order. So the core -//! installs committed transactions, and emits their change events, in -//! sequencer order on every replica. The commit tail, which publishes the -//! transaction's Control-Plane changes, runs in the same order. A Calvin -//! change feed therefore never delivers a lower position after a higher one. -//! -//! Every wait points at a lower transaction: the scheduler takes input, and -//! grants locks, in sequencer order. So the turn order adds no wait cycle. - -use super::super::types::CommitState; -use super::commit_resolution_dispatch::CommitResolution; -use super::deferred::{DispatchOutcome, DispatchStep}; -use super::install_gate::FlushAdmission; -use super::scheduler::Scheduler; -use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; - -impl Scheduler { - /// Queue the committed flush of `txn_id` for its turn, then dispatch it - /// when its turn came. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn queue_flush( - &mut self, - txn_id: TxnId, - redo_lsn: Option, - ) { - if let Some(pending) = self.pending.get_mut(&txn_id) { - pending.commit_state = CommitState::AwaitingFlushTurn { redo_lsn }; - } - self.pump_flush_turn(); - } - - /// Dispatch the flush of the lowest unfinished transaction when it waits - /// for its turn. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn pump_flush_turn(&mut self) { - let Some(lowest) = self.lowest_unfinished() else { - return; - }; - let resolve_turn = self - .pending - .get(&lowest) - .is_some_and(|pending| pending.commit_state == CommitState::AwaitingResolveTurn); - if resolve_turn { - self.dispatch_resolve_turn(lowest); - return; - } - let Some(redo_lsn) = - self.pending - .get(&lowest) - .and_then(|pending| match pending.commit_state { - CommitState::AwaitingFlushTurn { redo_lsn } => Some(redo_lsn), - _ => None, - }) - else { - return; - }; - // The flush holds its data group's apply gate shared until its - // position is marked applied (see `super::install_gate`). - let permit = match self.admit_flush() { - FlushAdmission::Go(permit) => permit, - FlushAdmission::Wait | FlushAdmission::Retired => return, - }; - if let Some(pending) = self.pending.get_mut(&lowest) { - pending.install_permit = permit; - } - // A flush refused at capacity is parked for re-send. The txn awaits its - // flush response either way, so the state below is the same. - if let DispatchOutcome::Failed(error) = - self.dispatch_commit_resolution(lowest, CommitResolution::Flush { redo_lsn }) - { - self.fail_dispatch_step(lowest, DispatchStep::Flush, error); - return; - } - if let Some(pending) = self.pending.get_mut(&lowest) { - pending.commit_state = CommitState::AwaitingResolve { - committed: true, - redo_lsn, - }; - } - } - - /// Dispatch the resolve of `txn_id`, whose turn came. Its flush then - /// follows at once: it stays the lowest unfinished txn. - fn dispatch_resolve_turn(&mut self, txn_id: TxnId) { - if let DispatchOutcome::Failed(error) = self.dispatch_calvin_resolve(txn_id) { - self.fail_dispatch_step(txn_id, DispatchStep::Resolve, error); - return; - } - if let Some(pending) = self.pending.get_mut(&txn_id) { - pending.commit_state = CommitState::AwaitingRedoResolve; - } - } - - /// The lowest transaction of this vShard that has not finished: in - /// flight, blocked on a lock, or waiting at a dependent-read barrier. - fn lowest_unfinished(&self) -> Option { - let pending = self.pending.keys().next().copied(); - let barrier = self.dependent_barrier.keys().next().copied(); - let blocked = self - .blocked - .values() - .map(|blocked| TxnId::new(blocked.txn.epoch, blocked.txn.position)) - .min(); - // A multi-part txn holding its locks while its parts arrive is - // unfinished too: every later flush waits for it. - let awaiting_parts = self.parts.lowest_awaiting(); - [pending, barrier, blocked, awaiting_parts] - .into_iter() - .flatten() - .min() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ - build_test_scheduler_with_data_side, make_sequenced_txn, staged_pending, - }; - use nodedb_cluster::calvin::CalvinCompletionRegistry; - - /// A higher committed txn waits for the lower one; the lower one's - /// completion gives it its turn. - #[tokio::test] - async fn a_higher_flush_waits_for_every_lower_txn() { - let low = TxnId::new(5, 0); - let high = TxnId::new(5, 1); - let (mut scheduler, _dir, _data_side) = - build_test_scheduler_with_data_side(7, CalvinCompletionRegistry::new_detached()); - for txn_id in [low, high] { - scheduler.pending.insert( - txn_id, - staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id), - ); - } - - scheduler.queue_flush(high, None); - assert_eq!( - scheduler.pending.get(&high).map(|p| p.commit_state), - Some(CommitState::AwaitingFlushTurn { redo_lsn: None }), - "the lower txn is unfinished, so the higher flush waits" - ); - - scheduler.pending.remove(&low); - scheduler.pump_flush_turn(); - assert_eq!( - scheduler.pending.get(&high).map(|p| p.commit_state), - Some(CommitState::AwaitingResolve { - committed: true, - redo_lsn: None, - }), - "with the lower txn finished, the higher flush dispatches" - ); - } - - /// A committed TRUNCATE of rows resolves only at its turn: while a lower - /// txn of this vShard is unfinished, its resolve waits, so it reads every - /// row sequenced before it. - #[tokio::test] - async fn a_truncate_resolves_only_at_its_turn() { - let low = TxnId::new(5, 0); - let truncate = TxnId::new(5, 1); - let (mut scheduler, _dir, _data_side) = - build_test_scheduler_with_data_side(7, CalvinCompletionRegistry::new_detached()); - for txn_id in [low, truncate] { - scheduler.pending.insert( - txn_id, - staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id), - ); - } - if let Some(pending) = scheduler.pending.get_mut(&truncate) { - pending.flush_scope.resolve_at_turn = true; - pending.commit_state = CommitState::AwaitingResolveTurn; - } - - scheduler.pump_flush_turn(); - assert_eq!( - scheduler.pending.get(&truncate).map(|p| p.commit_state), - Some(CommitState::AwaitingResolveTurn), - "the lower txn is unfinished, so the resolve waits" - ); - - scheduler.pending.remove(&low); - scheduler.pump_flush_turn(); - assert_eq!( - scheduler.pending.get(&truncate).map(|p| p.commit_state), - Some(CommitState::AwaitingRedoResolve), - "with the lower txn finished, the resolve dispatches" - ); - } - - /// Every truncate of rows reads a whole collection at resolve; a point - /// write does not. A TRUNCATE's edge share records a cut, which reads no - /// stored edge, so it resolves without waiting for its turn. - #[test] - fn a_truncate_slice_resolves_at_its_turn() { - use nodedb_physical::physical_plan::{DocumentOp, GraphOp, PhysicalPlan}; - let collection = nodedb_types::QualifiedCollection::from_stored("c".to_string()); - let truncate = PhysicalPlan::Document(DocumentOp::Truncate { - collection: collection.clone(), - restart_identity: false, - resolved_sum_targets: Vec::new(), - declared_primary_key: None, - }); - let share = PhysicalPlan::Graph(GraphOp::TruncateEdges { - collection, - vshard: 3, - }); - use crate::control::cluster::calvin::scheduler::driver::types::FlushScope; - assert!(FlushScope::of_plans(std::slice::from_ref(&truncate)).resolve_at_turn); - assert!(!FlushScope::of_plans(std::slice::from_ref(&share)).resolve_at_turn); - assert!( - FlushScope::of_plans(&[truncate, share]).resolve_at_turn, - "the rows' vShard holds both, and its rows' truncate waits" - ); - assert!(!FlushScope::of_plans(&[]).resolve_at_turn); - } -} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs index 5b28e8cae..13ee42747 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs @@ -10,17 +10,17 @@ //! //! A halted scheduler: //! - keeps the stuck txn unapplied, with its locks and its `pending` entry; -//! - releases the collection gates of every txn with no flush in flight, so a -//! purge never waits on it. Such a txn checks its incarnations again, and -//! takes its gates back, before a later flush; +//! - releases the collection gates of every txn, so a purge never waits on +//! it. A leader's txn checks its incarnations again, and takes its gates +//! back, before it proposes its redo; //! - closes intake with [`IntakeClosure::ApplyHalted`]. The fan-out then drops //! new input and arms catch-up, which keeps the sequencer log retained; //! - stops the deferred re-send and the catch-up drain; -//! - keeps routing responses, verdicts, and promotions for other in-flight -//! txns. Each one holds locks disjoint from the stuck txn, so its outcome -//! does not depend on it. Peer vShards wait on its votes, and the per-position -//! applied gate keeps restart replay exact. A later infrastructure error on -//! another txn holds that txn the same way. +//! - keeps routing responses, verdicts, promotions, and apply events for +//! other in-flight txns. Each one holds locks disjoint from the stuck txn, +//! so its outcome does not depend on it. Peer vShards wait on its votes, +//! and the per-position applied gate keeps restart replay exact. A later +//! infrastructure error on another txn holds that txn the same way. //! //! The first cause wins. It logs one `error!` line, sets the //! `nodedb_calvin_apply_halted` gauge, and records the node-wide @@ -51,22 +51,26 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) enum HaltReason ResponseDisconnected, /// The resolve of a committed txn failed or returned an undecodable record. ResolveFailed, - /// The flush of a committed txn returned an error. - FlushFailed, - /// This replica failed to stage a txn whose verdict is COMMIT. + /// This leader failed to stage a txn whose verdict is COMMIT. LocalStageFailed, /// The surrogate catalog refused a coordinator-assigned identity. IdentityBindFailed, - /// A redo or `CalvinApplied` WAL append failed. - WalAppendFailed, /// The metadata group left this node before its apply reached a held /// txn's floor. MetadataGroupGone, - /// A committed txn's write-version record cannot be built. Read-set - /// validation reads these versions, so a lost record reports false-valid reads. - WriteVersionRecordFailed, /// A stage response carries no vote the scheduler can read. StageVoteInvalid, + /// A committed slice's redo entry cannot be built. The verdict is + /// COMMIT, so the txn cannot drop. + RedoProposeFailed, + /// The vShard's data group applied a committed slice's redo and refused + /// it for good. Every replica refuses the same entry. + RedoInstallRefused, + /// A committed slice's redo did not become durable on this replica. + RedoApplyFailed, + /// The vShard's core fail-stopped. It refuses every later request, the + /// committed install included. + CoreFailStopped, } impl HaltReason { @@ -77,13 +81,14 @@ impl HaltReason { Self::DispatchRefused => apply_halt_reason::DISPATCH_REFUSED, Self::ResponseDisconnected => apply_halt_reason::RESPONSE_DISCONNECTED, Self::ResolveFailed => apply_halt_reason::RESOLVE_FAILED, - Self::FlushFailed => apply_halt_reason::FLUSH_FAILED, Self::LocalStageFailed => apply_halt_reason::LOCAL_STAGE_FAILED, Self::IdentityBindFailed => apply_halt_reason::IDENTITY_BIND_FAILED, - Self::WalAppendFailed => apply_halt_reason::WAL_APPEND_FAILED, Self::MetadataGroupGone => apply_halt_reason::METADATA_GROUP_GONE, - Self::WriteVersionRecordFailed => apply_halt_reason::WRITE_VERSION_RECORD_FAILED, Self::StageVoteInvalid => apply_halt_reason::STAGE_VOTE_INVALID, + Self::RedoProposeFailed => apply_halt_reason::REDO_PROPOSE_FAILED, + Self::RedoInstallRefused => apply_halt_reason::REDO_INSTALL_REFUSED, + Self::RedoApplyFailed => apply_halt_reason::REDO_APPLY_FAILED, + Self::CoreFailStopped => apply_halt_reason::CORE_FAIL_STOPPED, } } @@ -102,16 +107,12 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) enum HaltStep { Stage, /// The resolve request of a committed txn. Resolve, - /// The flush request of a committed txn. - Flush, - /// The drop request of an aborted txn. + /// The proposal of a committed slice's redo. + RedoPropose, + /// The install of a committed slice's redo, from the data-group log. + RedoApply, + /// The drop request of a txn that ends with no log entry. Drop, - /// The write-version record of a committed txn. - WriteVersionRecord, - /// The `TransactionRedo` WAL append. - RedoAppend, - /// The `CalvinApplied` WAL append. - AppliedMarker, /// The surrogate identity binding before the stage dispatch. IdentityBind, /// The wait for this node's metadata apply to reach the txn's floor. @@ -126,11 +127,9 @@ impl HaltStep { match self { Self::Stage => "stage", Self::Resolve => "resolve", - Self::Flush => "flush", + Self::RedoPropose => "redo_propose", + Self::RedoApply => "redo_apply", Self::Drop => "drop", - Self::WriteVersionRecord => "write_version_record", - Self::RedoAppend => "redo_append", - Self::AppliedMarker => "applied_marker", Self::IdentityBind => "identity_bind", Self::MetadataHold => "metadata_hold", } @@ -143,14 +142,8 @@ impl HaltStep { match state { CommitState::Staged | CommitState::AwaitingVerdict => Self::Stage, CommitState::AwaitingRedoResolve | CommitState::AwaitingResolveTurn => Self::Resolve, - CommitState::AwaitingFlushTurn { .. } => Self::Flush, - CommitState::AwaitingResolve { - committed: true, .. - } => Self::Flush, - CommitState::AwaitingResolve { - committed: false, .. - } => Self::Drop, - CommitState::AwaitingVersionRecord => Self::WriteVersionRecord, + CommitState::AwaitingRedoApply { .. } | CommitState::Following => Self::RedoApply, + CommitState::AwaitingDrop => Self::Drop, } } } @@ -210,9 +203,6 @@ impl Scheduler { step: HaltStep, error: String, ) { - // The txn stays unapplied here, so restart replay must reach its redo - // record. - self.hold_redo_records(txn_id); let reason = if self.node_shutting_down() { HaltReason::Draining } else { @@ -241,7 +231,7 @@ impl Scheduler { "calvin scheduler: apply halted; holding another txn unapplied" ); } - self.release_gates_on_halt(txn_id); + self.release_gates_on_halt(); return; } @@ -288,7 +278,7 @@ impl Scheduler { .record(report.clone()); } self.halt.first = Some(ApplyHalt { reason, report }); - self.release_gates_on_halt(txn_id); + self.release_gates_on_halt(); } /// Whether the node shuts down: the shutdown watch fired, or the diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/install_gate.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/install_gate.rs index da4feba01..f195d9abe 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/install_gate.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/install_gate.rs @@ -1,46 +1,22 @@ // SPDX-License-Identifier: BUSL-1.1 -//! The scheduler's admission of a flush against its data group's snapshots. +//! The scheduler's view of its vShard's installed base. //! -//! A flush installs a committed transaction's slice on the vShard's core. -//! The data group's apply gate orders it against the group's snapshots: +//! A data-group snapshot install replaces the vShard's base and its applied +//! ledger. A scheduler started under an older base proposes nothing more: +//! the next scheduler reconcile starts it again from the installed state. //! -//! - A snapshot capture on this node holds the gate exclusive. A flush holds -//! it shared from its dispatch until its position is marked applied, so -//! the capture reads storage and applied positions that agree. -//! - A snapshot install on this node holds it exclusive too, and replaces -//! the vShard's base. A scheduler started under an older base installs -//! nothing more: the next scheduler reconcile starts it again from the -//! installed state. -//! -//! A flush the gate refuses waits for the gate's release and is tried again. - -use tokio::sync::watch; +//! The data group's apply gate orders every install against snapshot +//! captures and installs: a committed slice installs from the log, in the +//! apply loop, under that gate. use super::scheduler::Scheduler; use crate::control::security::auth_fence::cluster::group_of_vshard; use crate::control::state::SharedState; -/// What the gate says to one flush. -pub(in crate::control::cluster::calvin::scheduler::driver::core) enum FlushAdmission { - /// Dispatch, holding the permit until the position is marked applied. - /// `None` when no apply gate is wired here. - Go(Option), - /// A snapshot holds the gate: try again once it releases. - Wait, - /// A snapshot install replaced this vShard's base since this scheduler - /// started: install nothing. - Retired, -} - -/// The scheduler's view of its data group's apply gate. +/// The base generation a scheduler started under. pub(in crate::control::cluster::calvin::scheduler::driver::core) struct InstallGate { - /// The vShard's base generation this scheduler started under. generation: u64, - /// Changes each time a snapshot releases a group's gate. - released: Option>, - /// Whether a flush waits for a release. - waiting: bool, } impl InstallGate { @@ -51,72 +27,21 @@ impl InstallGate { ) -> Self { Self { generation: shared.calvin.bases.generation(vshard_id), - released: shared - .raft_apply_gates - .get() - .map(|gates| gates.subscribe_released()), - waiting: false, - } - } - - /// Admit one flush of `vshard_id`. - fn admit(&mut self, shared: &SharedState, vshard_id: u32) -> FlushAdmission { - if shared.calvin.bases.generation(vshard_id) != self.generation { - return FlushAdmission::Retired; - } - let Some(gates) = shared.raft_apply_gates.get() else { - return FlushAdmission::Go(None); - }; - let Ok(group_id) = group_of_vshard(shared, vshard_id) else { - // No routing table holds the vShard: no snapshot of it runs. - return FlushAdmission::Go(None); - }; - // Marked as seen before the try, so a release after it still wakes - // the wait. - if let Some(released) = self.released.as_mut() { - drop(released.borrow_and_update()); - } - match gates.try_apply(group_id) { - Some(permit) => { - self.waiting = false; - FlushAdmission::Go(Some(permit)) - } - None => { - self.waiting = true; - FlushAdmission::Wait - } } } - /// Whether a flush waits for the gate's release. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn is_waiting(&self) -> bool { - self.waiting - } - - /// Resolves once a snapshot released a group's gate since the last - /// admission. - pub(in crate::control::cluster::calvin::scheduler::driver::core) async fn released(&mut self) { - match self.released.as_mut() { - Some(released) => { - if released.changed().await.is_err() { - // The gates are gone: no snapshot holds one again. - self.waiting = false; - } - } - None => std::future::pending().await, - } + /// Whether a snapshot install replaced `vshard_id`'s base since this + /// scheduler started. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn is_retired( + &self, + shared: &SharedState, + vshard_id: u32, + ) -> bool { + shared.calvin.bases.generation(vshard_id) != self.generation } } impl Scheduler { - /// Admit the flush of the lowest unfinished transaction. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn admit_flush( - &mut self, - ) -> FlushAdmission { - let vshard_id = self.vshard_id; - self.install_gate.admit(&self.shared, vshard_id) - } - /// Record that this replica's Calvin state of the vShard has a hole, and /// make its data-group replica refuse log entries until a snapshot /// brings the state back. @@ -151,76 +76,30 @@ impl Scheduler { #[cfg(test)] mod tests { - use std::sync::{Arc, RwLock}; - use std::time::Duration; - - use nodedb_cluster::{GroupApplyGates, RoutingTable}; + use std::sync::Arc; use super::*; use crate::bridge::dispatch::Dispatcher; use crate::wal::WalManager; - /// A snapshot capture holds the group's gate exclusive. A flush that - /// starts meanwhile waits, and a capture that starts while a flush is - /// out waits until the flush marked its position and dropped its - /// permit. So every capture sees each vShard's flushes either whole or - /// not at all, on every core. A snapshot install that replaced the - /// vShard's base stops the scheduler started before it. - #[tokio::test] - async fn a_capture_never_sees_a_flush_half_applied() { + /// A snapshot install that replaced the vShard's base retires the + /// scheduler started before it. + #[test] + fn an_installed_base_retires_the_older_scheduler() { let dir = tempfile::tempdir().expect("tempdir"); let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("gate.wal")).expect("wal")); let (dispatcher, _sides) = Dispatcher::new(1, 64); - let mut state = SharedState::new(dispatcher, wal).expect("shared state"); - Arc::get_mut(&mut state) - .expect("sole owner of fresh state") - .cluster_routing = Some(Arc::new(RwLock::new(RoutingTable::uniform(2, &[1], 1)))); - let gates = Arc::new(GroupApplyGates::new()); + let state = SharedState::new(dispatcher, wal).expect("shared state"); let vshard_id = 0; - let group_id = group_of_vshard(&state, vshard_id).expect("the vShard has a group"); - gates.mount(group_id); - assert!(state.raft_apply_gates.set(Arc::clone(&gates)).is_ok()); - - let mut gate = InstallGate::new(&state, vshard_id); - let capture = gates.install(group_id).await; - assert!(matches!( - gate.admit(&state, vshard_id), - FlushAdmission::Wait - )); - assert!(gate.is_waiting()); - drop(capture); - tokio::time::timeout(Duration::from_secs(5), gate.released()) - .await - .expect("the capture's release wakes the waiting flush"); - - let FlushAdmission::Go(Some(permit)) = gate.admit(&state, vshard_id) else { - panic!("an open gate admits the flush with a permit"); - }; - assert!(!gate.is_waiting()); - let next_capture = { - let gates = Arc::clone(&gates); - tokio::spawn(async move { drop(gates.install(group_id).await) }) - }; - tokio::time::sleep(Duration::from_millis(50)).await; - assert!( - !next_capture.is_finished(), - "a capture waits for the flush in flight" - ); - drop(permit); - tokio::time::timeout(Duration::from_secs(5), next_capture) - .await - .expect("the capture starts once the flush finished") - .expect("capture task"); + let gate = InstallGate::new(&state, vshard_id); + assert!(!gate.is_retired(&state, vshard_id)); state .calvin .bases .record_snapshot(state.credentials.catalog(), &[vshard_id], 9) .expect("record the installed base"); - assert!(matches!( - gate.admit(&state, vshard_id), - FlushAdmission::Retired - )); + assert!(gate.is_retired(&state, vshard_id)); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/intake.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/intake.rs index 35c30176e..4c45112e3 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/intake.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/intake.rs @@ -157,6 +157,7 @@ mod tests { use nodedb_types::TenantId; use tokio::sync::mpsc; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::lead_data_group; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ build_test_scheduler, build_test_scheduler_with_data_side, fill_tenant_inflight, make_sequenced_txn, make_validate_only_txn, release_filler, spawn_scheduler_loop, @@ -236,6 +237,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); let shared = Arc::clone(&scheduler.shared); let fillers = fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); scheduler @@ -317,12 +319,10 @@ mod tests { let (mut scheduler, _dir, _data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); let held = TxnId::new(3, 0); - scheduler - .pending - .insert(held, staged_pending(make_validate_only_txn(3, 0), held)); - scheduler - .handle_completion(held, RequestId::new(9), None) - .await; + let mut pending = staged_pending(make_validate_only_txn(3, 0), held); + pending.awaiting = Some(RequestId::new(9)); + scheduler.pending.insert(held, pending); + scheduler.handle_completion(held, RequestId::new(9), None); assert_eq!(scheduler.intake_closure(), Some(IntakeClosure::ApplyHalted)); let metrics = Arc::clone(&scheduler.metrics); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs index 3da3caa6d..47a2c510f 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs @@ -3,10 +3,13 @@ //! Calvin scheduler driver core. //! //! One [`Scheduler`] task runs per vshard hosted on this node. It receives -//! sequenced txns from the sequencer, acquires deterministic locks, dispatches -//! static / dependent-read transactions to the Data Plane, waits for executor -//! responses, and writes `CalvinApplied` WAL records. Each sub-module owns one -//! concern; see that file's own doc comment for what it does. +//! sequenced txns from the sequencer and acquires deterministic locks on +//! every replica. The vShard's data-group leader stages each granted txn on +//! the Data Plane, votes, and proposes a committed slice's stamped redo to +//! the data group. Every replica installs that redo from the log, and the +//! apply loop reports each install through the scheduler's inbox. Each +//! sub-module owns one concern; see that file's own doc comment for what it +//! does. //! //! All bookkeeping uses `BTreeMap`/`BTreeSet` — never `HashMap`/`HashSet` — //! and dispatch order is `(epoch, position)` order. `Instant::now()` is used @@ -14,15 +17,15 @@ //! `timeout_at`, both off the WAL-influencing path; every call site carries a //! `// no-determinism:` marker. +mod apply_result; pub mod catch_up; pub mod commit_redo; -pub mod commit_resolution_dispatch; pub mod commit_resolve; pub mod completion_route; mod cut_marker; pub mod deferred; pub mod dispatch; -pub mod flush_turn; +mod drop_dispatch; pub mod halt; mod install_gate; pub mod intake; @@ -34,8 +37,11 @@ pub mod parts_lane; pub mod process; pub mod propose; pub mod read_result; -mod redo_window; +pub mod redo_applied; +pub mod redo_propose; pub mod request; +pub mod resolve_turn; +pub mod role; pub mod routing; pub mod run_loop; pub mod scheduler; @@ -45,7 +51,6 @@ pub mod staged_vote; mod test_proposer; #[cfg(test)] mod test_support; -pub mod write_version_record; pub use propose::{CalvinReadResultProposal, propose_calvin_read_result}; pub use scheduler::{Scheduler, SchedulerParams}; diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/retry.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/retry.rs index e3e803523..4dc9142fd 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/retry.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/owed/retry.rs @@ -235,22 +235,14 @@ mod tests { #[tokio::test] async fn completion_ack_is_reproposed_until_the_registry_records_it() { let txn_id = TxnId::new(21, 0); - let (mut scheduler, _dir) = scheduler_with_pending( - txn_id, - CommitState::AwaitingResolve { - committed: false, - redo_lsn: None, - }, - ); + let (mut scheduler, _dir) = scheduler_with_pending(txn_id, CommitState::AwaitingDrop); let proposer = CapturingProposer::failing_first(1); scheduler.sequencer_proposer = proposer.clone(); // Only the vShard's group leader proposes the ack. elect_data_group_leader(&scheduler); scheduler.registry.seed_expected(cluster_txn_id(txn_id), 2); - scheduler - .finish_resolved_commit(txn_id, staged_response(Status::Ok, None), false, None) - .await; + scheduler.finish_drop(txn_id, &staged_response(Status::Ok, None)); assert!(!scheduler.pending.contains_key(&txn_id)); assert_eq!(proposer.attempt_count(), 1, "the first proposal is refused"); @@ -340,40 +332,6 @@ mod tests { assert_eq!(proposer.attempt_count(), 1); } - /// A follower that staged a txn owes an abort vote. It proposes the vote - /// only once it leads the vShard's group and no vote has applied, so the - /// txn ends in a retryable abort instead of waiting on a lost leader. - #[tokio::test] - async fn a_follower_that_comes_to_lead_votes_abort_for_its_staged_txn() { - let (mut scheduler, _dir) = build_test_scheduler(VSHARD); - let proposer = CapturingProposer::accepting(); - scheduler.sequencer_proposer = proposer.clone(); - let txn_id = TxnId::new(26, 0); - scheduler.registry.seed_expected(cluster_txn_id(txn_id), 2); - scheduler - .pending - .insert(txn_id, staged_pending(make_sequenced_txn(26, 0), txn_id)); - - scheduler.resolve_staged_commit( - txn_id, - &staged_response(Status::Ok, Some(StageVote::Commit)), - ); - scheduler.retry_owed_sequencer_entries(); - assert_eq!(proposer.attempt_count(), 0, "a follower casts no vote"); - - elect_data_group_leader(&scheduler); - scheduler.retry_owed_sequencer_entries(); - assert_eq!( - proposer.accepted(), - vec![SequencerEntry::AbortVote { - epoch: 26, - position: 0, - vshard: VSHARD, - reason: nodedb_cluster::calvin::AbortReason::SerializationConflict, - }] - ); - } - /// A txn this node never seeded keeps its entry owed: a missing registry /// entry that was never seen does not settle it. #[tokio::test] diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts.rs index 91fc66692..b82f19882 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts.rs @@ -23,7 +23,7 @@ //! every replica alike. //! - A txn whose locks are granted before its parts arrived waits in //! `awaiting`, holding its locks. It counts as unfinished, so every later -//! txn's flush on this vShard waits for it. +//! whole-collection resolve on this vShard waits for it. //! - Once every part that targets this vShard arrived, the txn stages as a //! single-entry txn whose plans are its local tasks. Its manifest keeps //! the whole-transaction facts the dispatch reads. @@ -235,6 +235,18 @@ impl PartsState { ) -> Option { self.awaiting.keys().next().copied() } + + /// Stop waiting for the parts of `txn_id`, which finished without them: + /// its redo installed from another replica's stage. Returns the owner of + /// the locks it holds. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn release_awaiting( + &mut self, + txn_id: TxnId, + ) -> Option { + let awaiting = self.awaiting.remove(&txn_id)?; + self.assemblies.remove(&txn_id); + Some(awaiting.lock_owner) + } } impl Scheduler { @@ -446,7 +458,7 @@ mod tests { use crate::bridge::envelope::Status; use crate::control::cluster::calvin::scheduler::driver::core::owed::OwedKind; use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ - CapturingProposer, elect_data_group_leader, + CapturingProposer, lead_data_group, }; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ build_test_scheduler_with_data_side, make_local_write_txn, staged_response, @@ -542,7 +554,7 @@ mod tests { ); let proposer = CapturingProposer::accepting(); scheduler.sequencer_proposer = proposer.clone(); - elect_data_group_leader(&scheduler); + lead_data_group(&mut scheduler); (scheduler, dir, data_side, proposer) } @@ -598,10 +610,7 @@ mod tests { scheduler.resume_on_verdict(txn_id, false); assert_eq!( scheduler.pending.get(&txn_id).map(|p| p.commit_state), - Some(CommitState::AwaitingResolve { - committed: false, - redo_lsn: None - }) + Some(CommitState::AwaitingDrop) ); assert!( !truncate_reached_data_plane(data_side), @@ -609,9 +618,7 @@ mod tests { ); assert!(!scheduler.applied.is_applied(txn_id.epoch, txn_id.position)); - scheduler - .finish_resolved_commit(txn_id, staged_response(Status::Ok, None), false, None) - .await; + scheduler.finish_drop(txn_id, &staged_response(Status::Ok, None)); assert!(!scheduler.pending.contains_key(&txn_id)); assert!(scheduler.applied.is_applied(txn_id.epoch, txn_id.position)); assert!( diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts_lane.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts_lane.rs index 041c34aec..7720c83f0 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts_lane.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/parts_lane.rs @@ -7,7 +7,8 @@ //! of the backlog, though, finishes only on input: //! //! - A multi-part txn that holds its locks finishes only once its parts -//! arrive, and every later flush on this vShard waits for it. +//! arrive, and every later whole-collection resolve on this vShard waits +//! for it. //! - A blocked txn below it can wait on a reservation release. //! //! While the gate is closed for the backlog, the lane lets exactly those @@ -41,7 +42,7 @@ //! parts and any abandonment of it follow its header in the log. They sit //! in the channel or in the log past the armed start, and the lane reaches //! both. So does a release a blocked txn waits on. The txn then stages, -//! flushes, and the backlog behind it drains, which opens the gate. +//! applies, and the backlog behind it drains, which opens the gate. use std::collections::VecDeque; diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs index fd236b620..d8455ebc5 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs @@ -13,6 +13,17 @@ use nodedb_cluster::calvin::types::{ use super::super::barrier::PendingDependentBarrier; use super::scheduler::Scheduler; + +/// Who records a finished position in the vShard's applied ledger. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::control::cluster::calvin::scheduler::driver::core) enum LedgerMark { + /// The position ended with no log entry: an abort, a slice with no + /// write, or an abandoned txn. The scheduler marks it. + Terminal, + /// The position's stamped redo installed. The apply loop marked it + /// before the group's applied index passed the entry. + ByApply, +} use crate::control::cluster::calvin::scheduler::lock_manager::{AcquireOutcome, TxnId}; /// Epochs a read reservation can live before the scheduler reaps it as orphaned. @@ -144,13 +155,12 @@ impl Scheduler { self.applied .note_expected(txn.epoch, txn.position, txn.epoch_vshard_txn_count); - // Exact per-position skip: never re-apply a position that already - // committed (its CalvinApplied marker is durable), and never re-run a - // whole epoch that has fully folded into the watermark. Re-running an - // applied position will re-fire its side effects — this gate IS the - // exactly-once mechanism. Skipping a whole epoch on its first completing - // position (the previous per-epoch gate) dropped every other position of - // that epoch across a restart: a torn transaction. + // Exact per-position skip: never re-run a position that already + // finished here (its stamped redo installed, or it ended with no log + // entry), and never re-run a whole epoch that has fully folded into + // the watermark. Re-running an applied position re-fires its side + // effects: this gate IS the exactly-once mechanism. A position whose + // redo applied before its input arrived is marked here already. if self.applied.is_applied(txn.epoch, txn.position) { // Learning the count for an already-applied position can complete a // historical epoch's applied set (during restart re-fan-out), folding @@ -221,17 +231,39 @@ impl Scheduler { } } - /// Route a ready txn to either a static dispatch or a dependent barrier. + /// Route a granted txn: a follower holds it, the leader stages it as a + /// static dispatch or a dependent barrier. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn dispatch_or_barrier( &mut self, txn: SequencedTxn, txn_id: TxnId, lock_owner: TxnId, ) { - // A multi-part txn dispatches only once its parts arrived. + // A multi-part txn dispatches only once its parts arrived. Every + // replica assembles them: a promotion stages the assembled txn. let Some(txn) = self.gate_on_parts(txn, txn_id, lock_owner) else { return; }; + // Its redo applied while it waited for its locks. + if let Some(event) = self.early_applied.remove(&txn_id) { + self.complete_applied(txn_id, lock_owner, event); + return; + } + self.route_granted(txn, txn_id, lock_owner); + } + + /// Route a granted, assembled txn: a follower holds it, the leader + /// stages it as a static dispatch or a dependent barrier. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn route_granted( + &mut self, + txn: SequencedTxn, + txn_id: TxnId, + lock_owner: TxnId, + ) { + if !self.role.is_leader() { + self.follow(txn, txn_id, lock_owner); + return; + } let is_dependent = txn.tx_class.dependent_reads.is_some(); if is_dependent { self.insert_dependent_barrier(txn, txn_id, lock_owner); @@ -267,16 +299,17 @@ impl Scheduler { self.dependent_barrier.insert(txn_id, barrier); } - /// Complete an in-flight txn (success or infrastructure error). + /// Complete an in-flight txn. /// /// Releases the lock-table owner recorded in its `pending` entry. A txn /// with no `pending` entry has already completed, so this logs and /// releases nothing. A multi-part txn whose parts the sequencer abandoned /// never enters `pending`, and uses [`Self::on_unpending_txn_complete`] - /// instead. + /// instead. `mark` says who records the position in the ledger. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn on_txn_complete( &mut self, txn_id: TxnId, + mark: LedgerMark, ) { let Some(pending) = self.pending.remove(&txn_id) else { tracing::error!( @@ -287,16 +320,7 @@ impl Scheduler { ); return; }; - // The flush applied the txn's redo record. A txn with no record - // finished without an install on this vShard. - let installed = match pending.redo_records { - Some(records) => { - records.settle(); - true - } - None => false, - }; - self.release_and_mark_applied(txn_id, pending.lock_owner, installed); + self.release_and_mark_applied(txn_id, pending.lock_owner, mark); } /// Complete a txn that never entered `pending`, releasing the locks held @@ -307,13 +331,17 @@ impl Scheduler { txn_id: TxnId, lock_owner: TxnId, ) { - self.release_and_mark_applied(txn_id, lock_owner, false); + self.release_and_mark_applied(txn_id, lock_owner, LedgerMark::Terminal); } /// Release `lock_owner`'s locks, dispatch the promoted waiters, and mark - /// `txn_id`'s position applied. `installed` says whether a durable redo - /// install finished the position. - fn release_and_mark_applied(&mut self, txn_id: TxnId, lock_owner: TxnId, installed: bool) { + /// `txn_id`'s position applied. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn release_and_mark_applied( + &mut self, + txn_id: TxnId, + lock_owner: TxnId, + mark: LedgerMark, + ) { // Release this txn's locks. `release` promotes any waiter queued behind // each freed key to holder (moving it pending -> held) and returns the // fully-promoted ids. Those ids are already holders in the table the @@ -332,16 +360,15 @@ impl Scheduler { // so any advertised watermark reflects a FULLY-applied epoch — the value // `BEGIN` needs for a torn-free cross-shard snapshot anchor. let folded = self.applied.mark_applied(txn_id.epoch, txn_id.position); - if installed { - self.ledger.mark_applied(txn_id.epoch, txn_id.position); - } else { - self.ledger.mark_terminal(txn_id.epoch, txn_id.position); + match mark { + LedgerMark::Terminal => self.ledger.mark_terminal(txn_id.epoch, txn_id.position), + LedgerMark::ByApply => {} } if let Some(watermark) = folded { self.publish_watermark(watermark); } - // A finished txn can give the next committed flush its turn. - self.pump_flush_turn(); + // A finished txn can give the next whole-collection resolve its turn. + self.pump_resolve_turn(); } /// Dispatch transactions that a `LockManager::release` promoted to holder. @@ -430,6 +457,7 @@ mod tests { use nodedb_physical::physical_plan::meta::MetaOp; use nodedb_types::TenantId; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::lead_data_group; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ await_data_plane_request, build_test_scheduler, build_test_scheduler_with_data_side, fill_tenant_inflight, make_sequenced_txn, make_validate_only_txn, release_filler, @@ -443,6 +471,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); let shared = Arc::clone(&scheduler.shared); fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); let watermark_before = shared.calvin.last_applied_epoch.load(Ordering::Acquire); @@ -468,6 +497,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); let shared = Arc::clone(&scheduler.shared); fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); @@ -488,6 +518,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); let shared = Arc::clone(&scheduler.shared); fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); let tracked_before = shared.tracker.in_flight(); @@ -509,6 +540,7 @@ mod tests { let registry = CalvinCompletionRegistry::new_detached(); let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side(test_coll_vshard(), registry); + lead_data_group(&mut scheduler); let shared = Arc::clone(&scheduler.shared); let fillers = fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs index 6e8815b6d..92bfbbd46 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs @@ -108,7 +108,7 @@ mod tests { use super::*; use crate::control::cluster::calvin::scheduler::driver::barrier::PendingDependentBarrier; use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ - CapturingProposer, elect_data_group_leader, + CapturingProposer, lead_data_group, }; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ build_test_scheduler_with_data_side, make_sequenced_txn, @@ -127,7 +127,7 @@ mod tests { build_test_scheduler_with_data_side(VSHARD, std::sync::Arc::clone(®istry)); let proposer = CapturingProposer::accepting(); scheduler.sequencer_proposer = proposer.clone(); - elect_data_group_leader(&scheduler); + lead_data_group(&mut scheduler); let txn_id = TxnId::new(40, 0); let cluster_txn = nodedb_cluster::calvin::TxnId::new(40, 0); registry.seed_expected(cluster_txn, 1); @@ -179,10 +179,7 @@ mod tests { scheduler.resume_on_verdict(txn_id, false); assert_eq!( scheduler.pending.get(&txn_id).map(|p| p.commit_state), - Some(CommitState::AwaitingResolve { - committed: false, - redo_lsn: None - }) + Some(CommitState::AwaitingDrop) ); assert!(!scheduler.applied.is_applied(40, 0)); } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_applied.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_applied.rs new file mode 100644 index 000000000..1728f46bf --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_applied.rs @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! What the scheduler does when the data-group apply loop concluded a +//! committed slice's stamped redo. Every replica runs it alike. +//! +//! - `RedoApplied` for a txn this scheduler holds: deposit the reply, +//! owe the `CompletionAck`, release the locks, mark the gate, complete. +//! The ack's bytes come from the reply, which every replica renders the +//! same, so the first ack the sequencer log holds is the true one. +//! - `RedoApplied` for a txn still waiting for its locks: kept until the +//! locks are granted, then completed the same way. +//! - `RedoApplied` for a txn whose input has not arrived: the log ran ahead +//! of the sequencer fan-out. Mark the gate and owe the ack. The input is +//! skipped when it arrives. +//! - `RedoRefused` and `RedoNotApplied` halt the scheduler. The txn holds +//! its locks. +//! +//! The apply loop marked the ledger before it pushed the event. A halted +//! scheduler still drains its inbox, so the apply loop never waits on it. + +use crate::bridge::envelope::Response; +use crate::control::cluster::calvin::scheduler::CalvinApplyEvent; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; +use crate::control::state::AckSlice; + +use super::super::types::CommitState; +use super::halt::{HaltReason, HaltStep}; +use super::owed::SchedulerProposal; +use super::process::LedgerMark; +use super::scheduler::Scheduler; + +/// A committed slice's install that is durable on this replica. +#[derive(Debug, Clone)] +pub(in crate::control::cluster::calvin::scheduler::driver::core) struct AppliedRedo { + pub reply: Response, + pub primary_write: bool, + pub returning: bool, +} + +impl Scheduler { + /// Handle every event waiting in this scheduler's inbox, in + /// `(epoch, position)` order. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn drain_inbox(&mut self) { + for ((epoch, position), event) in self.inbox.inbox().take_all() { + self.handle_calvin_apply_event(TxnId::new(epoch, position), event); + } + } + + /// Handle how the apply loop concluded the stamped redo of `txn_id`. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn handle_calvin_apply_event( + &mut self, + txn_id: TxnId, + event: CalvinApplyEvent, + ) { + match event { + CalvinApplyEvent::RedoApplied { + reply, + primary_write, + returning, + } => self.on_redo_applied( + txn_id, + AppliedRedo { + reply, + primary_write, + returning, + }, + ), + CalvinApplyEvent::RedoRefused { error } => self.halt_apply( + txn_id, + HaltReason::RedoInstallRefused, + HaltStep::RedoApply, + format!("the data group refused the committed slice's redo for good: {error}"), + ), + CalvinApplyEvent::RedoNotApplied { error } => self.halt_apply( + txn_id, + HaltReason::RedoApplyFailed, + HaltStep::RedoApply, + format!("the committed slice's redo did not become durable here: {error}"), + ), + } + } + + /// The stamped redo of `txn_id` installed on this replica. + fn on_redo_applied(&mut self, txn_id: TxnId, applied: AppliedRedo) { + if let Some(pending) = self.pending.get(&txn_id) { + let state = pending.commit_state; + let lock_owner = pending.lock_owner; + if holds_staged_state(state) { + // Another leader's copy installed while this node staged the + // txn again: the staged state is spent. + self.discard_staged(txn_id); + } + self.pending.remove(&txn_id); + self.complete_applied(txn_id, lock_owner, applied); + return; + } + if let Some(barrier) = self.dependent_barrier.remove(&txn_id) { + self.complete_applied(txn_id, barrier.lock_owner, applied); + return; + } + if let Some(lock_owner) = self.parts.release_awaiting(txn_id) { + self.complete_applied(txn_id, lock_owner, applied); + return; + } + let blocked = self + .blocked + .values() + .any(|blocked| TxnId::new(blocked.txn.epoch, blocked.txn.position) == txn_id); + if blocked { + self.early_applied.insert(txn_id, applied); + return; + } + // The input has not arrived. Its position is applied: the input is + // skipped when it does, and takes no lock. + self.owe_applied_ack(txn_id, &applied); + if let Some(watermark) = self.applied.mark_applied(txn_id.epoch, txn_id.position) { + self.publish_watermark(watermark); + } + } + + /// Complete `txn_id`, whose redo installed, releasing the locks + /// `lock_owner` holds. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn complete_applied( + &mut self, + txn_id: TxnId, + lock_owner: TxnId, + applied: AppliedRedo, + ) { + self.owe_applied_ack(txn_id, &applied); + self.metrics.record_completed(); + self.release_and_mark_applied(txn_id, lock_owner, LedgerMark::ByApply); + } + + /// Deposit a primary slice's reply, then owe the ack that carries it. + /// The deposit lands first, so the coordinator finds it when the ack + /// fires its waiter. + fn owe_applied_ack(&mut self, txn_id: TxnId, applied: &AppliedRedo) { + if applied.primary_write { + self.deposit_reply(txn_id, &applied.reply, applied.returning); + } + let result = self.ack_result_of( + &applied.reply, + AckSlice { + primary_write: applied.primary_write, + returning: applied.returning, + }, + ); + self.propose_sequencer_entry(txn_id, SchedulerProposal::CompletionAck { result }); + } +} + +/// Whether a txn in `state` holds staged state on this node's core. +fn holds_staged_state(state: CommitState) -> bool { + match state { + CommitState::Staged + | CommitState::AwaitingVerdict + | CommitState::AwaitingResolveTurn + | CommitState::AwaitingRedoResolve + | CommitState::AwaitingDrop => true, + // The install consumed the staged entry of a proposed slice, and a + // follower staged nothing. + CommitState::AwaitingRedoApply { .. } | CommitState::Following => false, + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use nodedb_cluster::calvin::CalvinCompletionRegistry; + use nodedb_cluster::calvin::types::SchedulerInput; + + use super::*; + use crate::bridge::envelope::Status; + use crate::control::cluster::calvin::scheduler::driver::core::halt::HaltReason; + use crate::control::cluster::calvin::scheduler::driver::core::owed::OwedKind; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler_with_data_side, make_sequenced_txn, redo_applied, + scheduler_with_pending, spawn_scheduler_loop, staged_response, test_coll_vshard, + }; + + /// The installed slice's reply. + fn reply() -> Response { + staged_response(Status::Ok, None) + } + + /// A follower's held txn whose redo installed completes: it owes its + /// ack, releases its locks, and marks its position applied. The txn + /// waiting on its locks runs. + #[tokio::test] + async fn redo_applied_releases_locks_and_owes_the_ack() { + let (mut scheduler, _dir, _data_side) = build_test_scheduler_with_data_side( + test_coll_vshard(), + CalvinCompletionRegistry::new_detached(), + ); + let txn_id = TxnId::new(3, 0); + let next = TxnId::new(4, 0); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(make_sequenced_txn(3, 0)))); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(make_sequenced_txn(4, 0)))); + assert!(scheduler.blocked.contains_key(&next), "the same key waits"); + + scheduler.handle_calvin_apply_event(txn_id, redo_applied(reply())); + + assert!(!scheduler.pending.contains_key(&txn_id)); + assert!(scheduler.applied.is_applied(3, 0)); + assert!( + scheduler + .owed + .contains_key(&(txn_id, OwedKind::CompletionAck)) + ); + assert!(!scheduler.blocked.contains_key(&next)); + assert!( + scheduler.pending.contains_key(&next), + "the freed locks grant the next txn" + ); + assert!(!scheduler.is_apply_halted()); + } + + /// The log ran ahead of the sequencer fan-out: the redo installed before + /// this scheduler saw the txn. Its position is applied and its ack owed, + /// and the input is skipped when it arrives. + #[tokio::test] + async fn redo_applied_before_the_input_skips_the_input() { + let (mut scheduler, _dir, _data_side) = build_test_scheduler_with_data_side( + test_coll_vshard(), + CalvinCompletionRegistry::new_detached(), + ); + let txn_id = TxnId::new(3, 0); + + scheduler.handle_calvin_apply_event(txn_id, redo_applied(reply())); + assert!(scheduler.applied.is_applied(3, 0)); + assert!( + scheduler + .owed + .contains_key(&(txn_id, OwedKind::CompletionAck)) + ); + + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(make_sequenced_txn(3, 0)))); + assert!( + !scheduler.pending.contains_key(&txn_id), + "the input is skipped" + ); + assert!(!scheduler.blocked.contains_key(&txn_id)); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(make_sequenced_txn(4, 0)))); + assert!( + scheduler.pending.contains_key(&TxnId::new(4, 0)), + "the skipped input took no lock" + ); + } + + /// A redo every replica refused for good halts the scheduler. The txn + /// keeps its locks and its position stays unapplied. + #[tokio::test] + async fn redo_refused_halts() { + let txn_id = TxnId::new(5, 1); + let (mut scheduler, _dir) = + scheduler_with_pending(txn_id, CommitState::AwaitingRedoApply { proposed: None }); + + scheduler.handle_calvin_apply_event( + txn_id, + CalvinApplyEvent::RedoRefused { + error: "constraint".to_string(), + }, + ); + + assert_eq!( + scheduler.apply_halt().map(|h| h.reason), + Some(HaltReason::RedoInstallRefused) + ); + assert!(scheduler.pending.contains_key(&txn_id)); + assert!(!scheduler.applied.is_applied(5, 1)); + } + + /// A halted scheduler still drains its inbox, so the apply loop never + /// waits on it. + #[tokio::test] + async fn a_halted_scheduler_still_drains_the_inbox() { + let txn_id = TxnId::new(5, 1); + let (mut scheduler, _dir) = + scheduler_with_pending(txn_id, CommitState::AwaitingRedoApply { proposed: None }); + scheduler.handle_calvin_apply_event( + txn_id, + CalvinApplyEvent::RedoNotApplied { + error: "fsync".to_string(), + }, + ); + assert!(scheduler.is_apply_halted()); + let inbox = std::sync::Arc::clone(scheduler.inbox.inbox()); + + let running = spawn_scheduler_loop(scheduler); + inbox + .push( + 6, + 0, + CalvinApplyEvent::RedoNotApplied { + error: "fsync".to_string(), + }, + ) + .await; + let drained = tokio::time::timeout(Duration::from_secs(5), async { + while inbox.waiting() > 0 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await; + running.stop().await; + + assert!(drained.is_ok(), "the halted scheduler drains its inbox"); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs new file mode 100644 index 000000000..64f8e8434 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs @@ -0,0 +1,450 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The data-group leader proposes a committed Calvin slice's stamped redo. +//! +//! The leader proposes right after the resolve, straight to its group with +//! `MultiRaft::propose`: no vShard admission slot and no forward. The +//! leader's write gate exempts a stamped redo, and the scheduler's lock +//! table already orders it against every conflicting write. Entries of +//! non-conflicting txns can enter the log in any order. +//! +//! A redo whose entry passes `tuning.calvin.max_redo_entry_bytes` travels +//! as chunk entries under a stream that names its position and attempt, +//! then a final entry naming the stream. +//! +//! The txn completes once its redo applies. Until then: +//! +//! | Outcome | Action | +//! |---|---| +//! | `NotLeader` | Demote. The next leader stages the txn again. | +//! | Proposed, then the group applied past the entry with no install | Propose again. | +//! | A transient refusal | Propose again on the stall tick. | +//! | The entry cannot be built | Halt: the verdict is COMMIT, so the txn cannot drop. | +//! +//! Every copy that commits installs at most once: the apply loop claims the +//! position in the vShard's applied ledger. + +use nodedb_cluster::ClusterError; +use nodedb_raft::RaftError; + +use super::super::types::CommitState; +use super::halt::{HaltReason, HaltStep}; +use super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; +use crate::control::wal_replication::ReplicatedWrite; +use crate::control::wal_replication::encode::{ + CalvinEntryStamps, RedoEntryTarget, RedoProposal, calvin_redo_proposal, +}; +use crate::wal::RedoStreamId; + +/// A committed slice's redo, kept while it is proposed. +#[derive(Debug, Clone)] +pub(in crate::control::cluster::calvin::scheduler::driver) struct OwedRedo { + pub payload: crate::control::wal_replication::transaction_redo::TransactionRedoPayload, + pub stamps: CalvinEntryStamps, + pub target: RedoEntryTarget, + /// Proposals made so far. Each names its own chunk stream. + pub attempts: u32, +} + +/// Why a proposal did not reach the log. +enum ProposeRefusal { + /// This node does not lead the vShard's data group. + NotLeader, + /// The group cannot take it now. The stall tick proposes again. + Transient(String), + /// The entries cannot be built. + Unbuildable(String), +} + +impl Scheduler { + /// Propose the redo `txn_id` owes, and record where it landed. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn propose_calvin_redo( + &mut self, + txn_id: TxnId, + ) { + if self + .pending + .get(&txn_id) + .and_then(|pending| pending.redo.as_ref()) + .is_none() + { + return; + } + #[cfg(feature = "failpoints")] + if self.redo_propose_held(txn_id) { + return; + } + // A snapshot install replaced this vShard's base since this + // scheduler started. The next scheduler starts from the installed + // state and stages the txn again. + if self.install_gate.is_retired(&self.shared, self.vshard_id) { + return; + } + // A halt released the txn's gates: take them back, or refuse a redo + // into a recreated collection. + if let Err(error) = self.regate(txn_id) { + self.halt_apply( + txn_id, + HaltReason::RedoProposeFailed, + HaltStep::RedoPropose, + error.to_string(), + ); + return; + } + let proposed = self + .attempt_entries(txn_id) + .and_then(|entries| self.propose_entries(entries)); + match proposed { + Ok(proposed) => { + if let Some(pending) = self.pending.get_mut(&txn_id) { + pending.commit_state = CommitState::AwaitingRedoApply { + proposed: Some(proposed), + }; + } + } + Err(ProposeRefusal::NotLeader) => self.on_not_leader(), + Err(ProposeRefusal::Transient(error)) => { + tracing::debug!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + %error, + "calvin: redo not proposed; the stall tick proposes it again" + ); + if let Some(pending) = self.pending.get_mut(&txn_id) { + pending.commit_state = CommitState::AwaitingRedoApply { proposed: None }; + } + } + Err(ProposeRefusal::Unbuildable(error)) => self.halt_apply( + txn_id, + HaltReason::RedoProposeFailed, + HaltStep::RedoPropose, + format!("the committed slice's redo entry cannot be built: {error}"), + ), + } + } + + /// Whether a crash test holds `txn_id`'s redo: it resolved and no copy + /// entered the log. The stall tick asks again. + #[cfg(feature = "failpoints")] + fn redo_propose_held(&self, txn_id: TxnId) -> bool { + let held = self + .pending + .get(&txn_id) + .and_then(|pending| pending.redo.as_ref()) + .is_some_and(|owed| { + crate::control::fail_gate::holds_redo_propose(&owed.payload.collections) + }); + if held { + tracing::info!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + "calvin: redo proposal held at a fail point" + ); + } + held + } + + /// The encoded entries of the next proposal attempt of `txn_id`. + fn attempt_entries(&mut self, txn_id: TxnId) -> Result>, ProposeRefusal> { + let max_entry_bytes = self.shared.redo_chunks.limits().max_entry_bytes; + let vshard = self.vshard_id; + let until = self.request_deadline(); + let Some(owed) = self + .pending + .get_mut(&txn_id) + .and_then(|pending| pending.redo.as_mut()) + else { + return Err(ProposeRefusal::Unbuildable( + "the txn owes no redo".to_owned(), + )); + }; + let stream = RedoStreamId::Calvin { + vshard, + epoch: txn_id.epoch, + position: txn_id.position, + attempt: owed.attempts, + }; + owed.attempts = owed.attempts.saturating_add(1); + let unbuildable = |error: crate::Error| ProposeRefusal::Unbuildable(error.to_string()); + let proposal = calvin_redo_proposal( + owed.target, + &owed.payload, + &owed.stamps, + stream, + max_entry_bytes, + ) + .map_err(unbuildable)?; + let (chunks, last) = match proposal { + RedoProposal::Inline(last) => (Vec::new(), last), + RedoProposal::Chunked { + stream, + chunks, + last, + } => { + let len = if let Some(ReplicatedWrite::RedoChunk { len, .. }) = + chunks.first().map(|chunk| &chunk.write) + { + *len + } else { + 0 + }; + // The leader bounds the node's open stream bytes: a stream + // past the cap waits for the stall tick. + self.shared + .redo_chunks + .admit_stream(stream, len, until) + .map_err(|refusal| ProposeRefusal::Transient(refusal.to_string()))?; + (chunks, last) + } + }; + let mut entries = Vec::with_capacity(chunks.len() + 1); + for entry in chunks.iter().chain(std::iter::once(&last)) { + entries.push(entry.encode().map_err(unbuildable)?); + } + Ok(entries) + } + + /// Append `entries` to this vShard's data-group log, in order. Returns + /// `(group_id, log_index)` of the last. + fn propose_entries(&self, entries: Vec>) -> Result<(u64, u64), ProposeRefusal> { + let mut mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + let mut last = None; + for bytes in entries { + match mr.propose(self.vshard_id, bytes) { + Ok(at) => last = Some(at), + Err(ClusterError::Raft(RaftError::NotLeader { .. })) => { + return Err(ProposeRefusal::NotLeader); + } + Err(error) => return Err(ProposeRefusal::Transient(error.to_string())), + } + } + last.ok_or_else(|| ProposeRefusal::Unbuildable("a redo with no entry".to_owned())) + } + + /// Propose again every owed redo that is not in the log: never + /// proposed, refused, or proposed and passed by the group's applied + /// index with no install, which a new term's entries overwrote. Runs + /// on the stall tick, after the inbox drained, on the leader only. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn retry_redo_proposals( + &mut self, + ) { + if !self.role.is_leader() { + return; + } + let owed: Vec<(TxnId, Option<(u64, u64)>)> = self + .pending + .iter() + .filter_map(|(txn_id, pending)| match pending.commit_state { + CommitState::AwaitingRedoApply { proposed } => Some((*txn_id, proposed)), + CommitState::Following + | CommitState::Staged + | CommitState::AwaitingVerdict + | CommitState::AwaitingResolveTurn + | CommitState::AwaitingRedoResolve + | CommitState::AwaitingDrop => None, + }) + .collect(); + for (txn_id, proposed) in owed { + let lost = match proposed { + None => true, + Some((group_id, log_index)) => { + self.shared.applied_index_watcher(group_id).current() >= log_index + } + }; + if lost { + if proposed.is_some() { + tracing::warn!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + "calvin: the group applied past the slice's redo with no install; \ + proposing it again" + ); + } + self.propose_calvin_redo(txn_id); + } + if !self.role.is_leader() { + return; + } + } + } +} + +#[cfg(test)] +mod tests { + use nodedb_cluster::calvin::CalvinCompletionRegistry; + + use super::*; + use crate::bridge::dispatch::CoreChannelDataSide; + use crate::bridge::envelope::{Payload, Response, Status}; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ + lead_data_group, step_down_from_data_group, + }; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler_with_data_side, make_sequenced_txn, staged_pending, staged_response, + }; + use crate::control::wal_replication::ReplicatedEntry; + use crate::wal::RedoRecord; + + const VSHARD: u32 = 7; + + /// A leader on [`VSHARD`] with `txn_id` committed and its resolve + /// awaited. + fn resolving_leader(txn_id: TxnId) -> (Scheduler, tempfile::TempDir, CoreChannelDataSide) { + let (mut scheduler, dir, data_side) = + build_test_scheduler_with_data_side(VSHARD, CalvinCompletionRegistry::new_detached()); + lead_data_group(&mut scheduler); + let mut pending = staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id); + pending.commit_state = CommitState::AwaitingRedoResolve; + scheduler.pending.insert(txn_id, pending); + (scheduler, dir, data_side) + } + + /// A resolve answer whose redo record carries no ops. + fn resolved_answer() -> Response { + let redo = RedoRecord { + version: 1, + ops: Vec::new(), + calvin_stamp: None, + cross_shard_applied: None, + row_sources: Vec::new(), + publishes: Vec::new(), + row_changes: Vec::new(), + }; + let resolved = nodedb_physical::physical_plan::CalvinResolved { + redo: redo.to_bytes().expect("encode redo"), + reply: nodedb_physical::physical_plan::CalvinReplySpec::Count(Vec::new()), + }; + let mut response = staged_response(Status::Ok, None); + response.payload = + Payload::from_vec(zerompk::to_msgpack_vec(&resolved).expect("encode resolved")); + response + } + + /// Where `txn_id`'s redo entry landed, if it is in the log. + fn proposed_at(scheduler: &Scheduler, txn_id: TxnId) -> Option<(u64, u64)> { + match scheduler.pending.get(&txn_id).map(|p| p.commit_state) { + Some(CommitState::AwaitingRedoApply { proposed }) => proposed, + other => panic!("the txn awaits its redo apply, found {other:?}"), + } + } + + /// The committed entry of `group_id` at `index`. Ticks the single-voter + /// group until the entry commits. + fn committed_entry(scheduler: &Scheduler, group_id: u64, index: u64) -> Vec { + let mut mr = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()); + for _ in 0..20 { + let entries = mr + .read_committed_entries(group_id, index, index) + .expect("read the group's log"); + if let Some(entry) = entries.into_iter().find(|entry| entry.index == index) { + return entry.data; + } + mr.wait_all_durable_blocking(); + mr.tick().expect("tick"); + } + panic!("the entry at {index} never committed"); + } + + /// A committed write slice's resolve proposes exactly one entry: the + /// stamped redo of the slice, on the vShard's data group. + #[tokio::test] + async fn a_committed_write_slice_proposes_one_stamped_entry() { + let txn_id = TxnId::new(14, 2); + let (mut scheduler, _dir, _data_side) = resolving_leader(txn_id); + let group_id = { + let mr = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()); + mr.routing() + .read() + .unwrap_or_else(|p| p.into_inner()) + .group_for_vshard(VSHARD) + .expect("the fixture routes every vShard") + }; + let last_before = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .last_log_index(group_id) + .expect("the group is mounted"); + + scheduler.finish_redo_resolve(txn_id, resolved_answer()); + + let (proposed_group, index) = + proposed_at(&scheduler, txn_id).expect("the redo is in the log"); + assert_eq!(proposed_group, group_id); + assert_eq!(index, last_before + 1, "one entry, no chunks"); + let entry = ReplicatedEntry::from_bytes(&committed_entry(&scheduler, group_id, index)) + .expect("a replicated entry"); + assert_eq!(entry.vshard_id, VSHARD); + assert!( + matches!( + entry.write, + ReplicatedWrite::TransactionRedo { + calvin: Some(_), + .. + } + ), + "the entry is the slice's stamped redo" + ); + assert!(!scheduler.applied.is_applied(14, 2)); + assert!(!scheduler.is_apply_halted()); + } + + /// The group applied past the proposed entry with no install: a new + /// term overwrote it. The stall tick proposes the redo again. + #[tokio::test] + async fn an_overwritten_proposal_is_proposed_again() { + let txn_id = TxnId::new(14, 2); + let (mut scheduler, _dir, _data_side) = resolving_leader(txn_id); + scheduler.finish_redo_resolve(txn_id, resolved_answer()); + let (group_id, first) = proposed_at(&scheduler, txn_id).expect("proposed"); + + scheduler.retry_redo_proposals(); + assert_eq!( + proposed_at(&scheduler, txn_id), + Some((group_id, first)), + "an entry the group has not applied past stays" + ); + + scheduler.shared.applied_index_watcher(group_id).bump(first); + scheduler.retry_redo_proposals(); + + let (_, second) = proposed_at(&scheduler, txn_id).expect("proposed again"); + assert!(second > first, "a new entry carries the redo"); + if let Some(pending) = scheduler.pending.get(&txn_id) { + assert_eq!( + pending.redo.as_ref().map(|owed| owed.attempts), + Some(2), + "each proposal names its own attempt" + ); + } + } + + /// A proposal refused because this node no longer leads demotes the + /// scheduler: the txn is held `Following` for the next leader. + #[tokio::test] + async fn not_leader_demotes() { + let txn_id = TxnId::new(14, 2); + let (mut scheduler, _dir, _data_side) = resolving_leader(txn_id); + step_down_from_data_group(&scheduler); + + scheduler.finish_redo_resolve(txn_id, resolved_answer()); + + assert!(!scheduler.role.is_leader()); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Following) + ); + assert!(!scheduler.is_apply_halted()); + assert!(!scheduler.applied.is_applied(14, 2)); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_window.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_window.rs deleted file mode 100644 index 04dab5e8b..000000000 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_window.rs +++ /dev/null @@ -1,124 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Close the outcome-floor window of a committed txn's `TransactionRedo` -//! record. -//! -//! The record joins the pending txn when it is appended. The window settles -//! when the txn completes: the flush applied the record. It holds when the -//! txn stays unapplied here, because restart replay must reach the record. - -use super::scheduler::Scheduler; -use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; - -impl Scheduler { - /// Hold the redo record of `txn_id`: the scheduler halted with the txn - /// unapplied. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn hold_redo_records( - &mut self, - txn_id: TxnId, - ) { - if let Some(records) = self - .pending - .get_mut(&txn_id) - .and_then(|pending| pending.redo_records.take()) - { - records.hold(); - } - } - - /// Hold the redo record of every pending txn: the scheduler stops with - /// them unapplied. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn hold_all_redo_records( - &mut self, - ) { - for pending in self.pending.values_mut() { - if let Some(records) = pending.redo_records.take() { - records.hold(); - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::control::cluster::calvin::scheduler::driver::core::halt::{HaltReason, HaltStep}; - use crate::control::cluster::calvin::scheduler::driver::core::test_support::scheduler_with_pending; - use crate::control::cluster::calvin::scheduler::driver::types::CommitState; - use crate::control::server::dispatch_utils::MintedRecords; - use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; - - /// Attach an appended redo record to the pending txn, as a committed - /// resolve does. - fn attach_redo_record(scheduler: &mut Scheduler, txn_id: TxnId) -> Lsn { - let records = MintedRecords::open(&scheduler.shared.outcome_floor); - let lsn = records - .appender(&scheduler.shared.wal, crate::wal::manager::NO_APPLY_KEY) - .with_event_source(crate::event::EventSource::User) - .append_put( - TenantId::new(1), - VShardId::new(0), - DatabaseId::DEFAULT, - b"redo", - ) - .expect("append"); - if let Some(pending) = scheduler.pending.get_mut(&txn_id) { - pending.redo_records = Some(records); - } - lsn - } - - fn awaiting_flush() -> CommitState { - CommitState::AwaitingResolve { - committed: true, - redo_lsn: None, - } - } - - #[tokio::test] - async fn a_halted_txn_holds_its_redo_record() { - let txn_id = TxnId::new(3, 1); - let (mut scheduler, _dir) = scheduler_with_pending(txn_id, awaiting_flush()); - let lsn = attach_redo_record(&mut scheduler, txn_id); - - scheduler.halt_apply( - txn_id, - HaltReason::FlushFailed, - HaltStep::Flush, - "flush refused".to_string(), - ); - - let floor = &scheduler.shared.outcome_floor; - assert!( - floor.floor() < lsn, - "the held record keeps the floor below it" - ); - assert_eq!(floor.leaked_windows(), 0); - } - - #[tokio::test] - async fn a_completed_txn_settles_its_redo_record() { - let txn_id = TxnId::new(3, 2); - let (mut scheduler, _dir) = scheduler_with_pending(txn_id, awaiting_flush()); - let lsn = attach_redo_record(&mut scheduler, txn_id); - - scheduler.on_txn_complete(txn_id); - - let floor = &scheduler.shared.outcome_floor; - assert_eq!(floor.floor(), lsn); - assert_eq!(floor.leaked_windows(), 0); - } - - #[tokio::test] - async fn a_stopping_scheduler_holds_every_pending_redo_record() { - let txn_id = TxnId::new(3, 3); - let (mut scheduler, _dir) = scheduler_with_pending(txn_id, awaiting_flush()); - let lsn = attach_redo_record(&mut scheduler, txn_id); - - scheduler.hold_all_redo_records(); - - let floor = &scheduler.shared.outcome_floor; - assert!(floor.floor() < lsn); - assert_eq!(floor.leaked_windows(), 0); - } -} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/request.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/request.rs index 95d79d73d..57f186302 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/request.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/request.rs @@ -7,7 +7,7 @@ use std::time::{Duration, Instant}; use super::scheduler::Scheduler; use crate::bridge::envelope::{Admission, ExemptReason, Priority, Request}; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; -use crate::types::{DatabaseId, Lsn, ReadConsistency, RequestId, TenantId, VShardId}; +use crate::types::{DatabaseId, ReadConsistency, RequestId, TenantId, VShardId}; use nodedb_physical::physical_plan::PhysicalPlan; /// The event source a Calvin sub-operation runs with when its transaction @@ -15,9 +15,9 @@ use nodedb_physical::physical_plan::PhysicalPlan; pub(in crate::control::cluster::calvin::scheduler::driver::core) const CALVIN_EVENT_SOURCE: crate::event::EventSource = crate::event::EventSource::User; -/// The event source a sequenced transaction's writes carry. Its redo record -/// and its flush both carry it, so WAL replay rebuilds the events the flush -/// emits, and a server-run body's writes fire no triggers. +/// The event source a sequenced transaction's writes carry. Its redo entry +/// carries it, so every replica's install and WAL replay emit the same +/// events, and a server-run body's writes fire no triggers. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn txn_event_source( tx_class: &nodedb_cluster::calvin::types::TxClass, ) -> crate::event::EventSource { @@ -27,14 +27,14 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) fn txn_event_so /// The event source this vShard's slice of a sequenced transaction commits /// under. A slice only trigger bodies wrote, in a transaction a client also /// wrote, takes the source a body's row takes in one overlay. Every other -/// slice takes the transaction's source. The stage, the resolve, the redo -/// record and the flush all carry it. +/// slice takes the transaction's source. The stage and the redo entry both +/// carry it. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn slice_event_source( tx_class: &nodedb_cluster::calvin::types::TxClass, - flush_scope: &super::super::types::FlushScope, + scope: &super::super::types::SliceScope, ) -> crate::event::EventSource { let source = txn_event_source(tx_class); - if flush_scope.body_only { + if scope.body_only { source.committed_row_override(crate::event::EventSource::Trigger) } else { source @@ -46,17 +46,16 @@ impl Scheduler { /// /// Calvin has already globally ordered the transaction, so the request is /// admitted `Exempt(AlreadyOrdered)` — it bypasses admission control. Only - /// `request_id`, `tenant_id`, `plan`, and `wal_lsn` vary between call sites; - /// everything else (vshard, deadline, and the fixed metadata constants) is - /// derived from `&self` or constant. `wal_lsn` is `Some` only when the caller - /// has already durably appended the record this request applies. + /// `request_id`, `tenant_id`, `database_id` and `plan` vary between call + /// sites; everything else (vshard, deadline, and the fixed metadata + /// constants) is derived from `&self` or constant. No scheduler request + /// writes a WAL record, so none carries a WAL LSN. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn build_exempt_request( &self, request_id: RequestId, tenant_id: TenantId, database_id: DatabaseId, plan: PhysicalPlan, - wal_lsn: Option, ) -> Request { Request { request_id, @@ -74,7 +73,7 @@ impl Scheduler { user_id: None, statement_digest: None, txn_id: None, - wal_lsn, + wal_lsn: None, resolved_now_ms: None, commit_hlc: None, admission: Admission::Exempt(ExemptReason::AlreadyOrdered), diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/resolve_turn.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/resolve_turn.rs new file mode 100644 index 000000000..291b93e62 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/resolve_turn.rs @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The resolve turn of a slice that reads a whole collection at resolve. +//! +//! A committed slice resolves as soon as its verdict arrives, and its redo +//! is proposed at once: the log orders it against every other entry of the +//! group, and conflicting txns never overlap under the lock table. A +//! TRUNCATE of rows is the exception. Its resolve reads every row of the +//! collection, so it waits until every lower txn of this vShard finished: +//! their rows are then installed, and no higher txn's rows are. +//! +//! Every wait points at a lower transaction: the scheduler takes input, and +//! grants locks, in sequencer order. So the turn order adds no wait cycle. + +use super::super::types::CommitState; +use super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + +impl Scheduler { + /// Resolve the lowest unfinished transaction when it waits for its turn. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn pump_resolve_turn( + &mut self, + ) { + let Some(lowest) = self.lowest_unfinished() else { + return; + }; + let at_turn = self + .pending + .get(&lowest) + .is_some_and(|pending| pending.commit_state == CommitState::AwaitingResolveTurn); + if at_turn { + self.start_resolve(lowest); + } + } + + /// The lowest transaction of this vShard that has not finished: in + /// flight, blocked on a lock, or waiting at a dependent-read barrier. + fn lowest_unfinished(&self) -> Option { + let pending = self.pending.keys().next().copied(); + let barrier = self.dependent_barrier.keys().next().copied(); + let blocked = self + .blocked + .values() + .map(|blocked| TxnId::new(blocked.txn.epoch, blocked.txn.position)) + .min(); + // A multi-part txn holding its locks while its parts arrive is + // unfinished too: every later whole-collection resolve waits for it. + let awaiting_parts = self.parts.lowest_awaiting(); + [pending, barrier, blocked, awaiting_parts] + .into_iter() + .flatten() + .min() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler_with_data_side, make_sequenced_txn, staged_pending, + }; + use crate::control::cluster::calvin::scheduler::driver::types::SliceScope; + use nodedb_cluster::calvin::CalvinCompletionRegistry; + + /// A committed TRUNCATE of rows resolves only at its turn: while a lower + /// txn of this vShard is unfinished, its resolve waits, so it reads every + /// row sequenced before it. + #[tokio::test] + async fn a_truncate_resolves_only_at_its_turn() { + let low = TxnId::new(5, 0); + let truncate = TxnId::new(5, 1); + let (mut scheduler, _dir, _data_side) = + build_test_scheduler_with_data_side(7, CalvinCompletionRegistry::new_detached()); + for txn_id in [low, truncate] { + scheduler.pending.insert( + txn_id, + staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id), + ); + } + if let Some(pending) = scheduler.pending.get_mut(&truncate) { + pending.scope.resolve_at_turn = true; + pending.commit_state = CommitState::AwaitingResolveTurn; + } + + scheduler.pump_resolve_turn(); + assert_eq!( + scheduler.pending.get(&truncate).map(|p| p.commit_state), + Some(CommitState::AwaitingResolveTurn), + "the lower txn is unfinished, so the resolve waits" + ); + + scheduler.pending.remove(&low); + scheduler.pump_resolve_turn(); + assert_eq!( + scheduler.pending.get(&truncate).map(|p| p.commit_state), + Some(CommitState::AwaitingRedoResolve), + "with the lower txn finished, the resolve dispatches" + ); + } + + /// Every truncate of rows reads a whole collection at resolve; a point + /// write does not. A TRUNCATE's edge share records a cut, which reads no + /// stored edge, so it resolves without waiting for its turn. + #[test] + fn a_truncate_slice_resolves_at_its_turn() { + use nodedb_physical::physical_plan::{DocumentOp, GraphOp, PhysicalPlan}; + let collection = nodedb_types::QualifiedCollection::from_stored("c".to_string()); + let truncate = PhysicalPlan::Document(DocumentOp::Truncate { + collection: collection.clone(), + restart_identity: false, + resolved_sum_targets: Vec::new(), + declared_primary_key: None, + }); + let share = PhysicalPlan::Graph(GraphOp::TruncateEdges { + collection, + vshard: 3, + }); + assert!(SliceScope::of_plans(std::slice::from_ref(&truncate)).resolve_at_turn); + assert!(!SliceScope::of_plans(std::slice::from_ref(&share)).resolve_at_turn); + assert!( + SliceScope::of_plans(&[truncate, share]).resolve_at_turn, + "the rows' vShard holds both, and its rows' truncate waits" + ); + assert!(!SliceScope::of_plans(&[]).resolve_at_turn); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/role.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/role.rs new file mode 100644 index 000000000..d702c9b28 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/role.rs @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! This scheduler's role in its vShard's data group. +//! +//! Only the data-group leader stages, votes, resolves, and proposes a +//! committed slice's redo. Every other replica holds each granted txn as +//! [`CommitState::Following`] and installs its slice from the log. +//! +//! A node that wins a term leads only once the term's election no-op +//! applied here: the stage gate. Every entry of an earlier term in its log +//! sits below the no-op and commits with it. So a redo an earlier leader +//! proposed installs before this leader stages anything, and this leader +//! finds that position applied. +//! +//! - Promotion stages every `Following` txn as a new grant, in sequencer +//! order. A txn whose verdict is already COMMIT proposes its vote again, +//! which changes nothing: the first vote of the vShard counts. +//! - Demotion discards the staged state of every txn the leader drove, +//! drops its owed votes, and holds the txn `Following`. A redo this node +//! proposed can still commit in the new term: the applied ledger installs +//! one copy per position. +//! +//! [`CommitState::Following`]: super::super::types::CommitState::Following + +use super::super::types::CommitState; +use super::deferred::{DispatchOutcome, DispatchStep}; +use super::owed::OwedKind; +use super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + +/// What this node does for its vShard's Calvin txns. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::control::cluster::calvin::scheduler::driver::core) enum Role { + /// Holds granted txns and installs committed slices from the log. + Follower, + /// Stages, votes, resolves, and proposes. + Leader, +} + +/// The stage gate: the role, and the leadership term it opened in. +#[derive(Debug, Clone, Copy)] +pub(in crate::control::cluster::calvin::scheduler::driver::core) struct StageGate { + role: Role, + /// The data-group term this node leads, as last seen. + term: Option, + /// The index of the term's election no-op. The gate opens once this + /// node applied it. + term_start: Option, +} + +impl Default for StageGate { + fn default() -> Self { + Self { + role: Role::Follower, + term: None, + term_start: None, + } + } +} + +impl StageGate { + /// Whether this node stages the vShard's txns. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn is_leader(&self) -> bool { + self.role == Role::Leader + } + + /// Whether this node leads the data group and waits for its term's + /// no-op to apply. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn awaits_term_start( + &self, + ) -> bool { + self.role == Role::Follower && self.term.is_some() + } +} + +/// What the data group says about this node now. +struct Leadership { + group_id: u64, + term: u64, + term_start: Option, +} + +impl Scheduler { + /// Read this node's place in the vShard's data group, and promote or + /// demote on a change. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn refresh_role(&mut self) { + let Some(leadership) = self.leadership() else { + self.role.term = None; + self.role.term_start = None; + if self.role.is_leader() { + self.demote("this node no longer leads the vShard's data group"); + } + return; + }; + if self.role.term != Some(leadership.term) { + if self.role.is_leader() { + self.demote("this node leads a later term of the vShard's data group"); + } + self.role.term = Some(leadership.term); + self.role.term_start = leadership.term_start; + } + if self.role.is_leader() { + return; + } + let applied = self + .shared + .applied_index_watcher(leadership.group_id) + .current(); + if self.role.term_start.is_some_and(|start| applied >= start) { + self.promote(); + } + } + + /// This node's term and term start in the vShard's data group, while it + /// leads the group. + fn leadership(&self) -> Option { + let mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + let group_id = mr + .routing() + .read() + .unwrap_or_else(|p| p.into_inner()) + .group_for_vshard(self.vshard_id) + .ok()?; + let term = mr.leader_term(group_id)?; + Some(Leadership { + group_id, + term, + term_start: mr.term_start_index(group_id), + }) + } + + /// Whether this node leads the vShard's data group in Raft right now, + /// whatever its stage gate says. Owed sequencer entries are proposed + /// only then. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn is_group_leader( + &self, + ) -> bool { + let mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + mr.vshard_role_is_leader(self.vshard_id) + } + + /// Open the stage gate: stage every `Following` txn, in sequencer order. + fn promote(&mut self) { + self.role.role = Role::Leader; + tracing::info!( + vshard_id = self.vshard_id, + term = self.role.term, + "calvin scheduler: this node leads the vShard; staging its held txns" + ); + let following: Vec = self + .pending + .iter() + .filter(|(_, pending)| pending.commit_state == CommitState::Following) + .map(|(txn_id, _)| *txn_id) + .collect(); + for txn_id in following { + let Some(pending) = self.pending.remove(&txn_id) else { + continue; + }; + self.route_granted(pending.txn, txn_id, pending.lock_owner); + } + } + + /// Close the stage gate: discard the staged state of every txn this + /// node drove, drop its owed votes, and hold each txn `Following`. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn demote( + &mut self, + why: &str, + ) { + self.role.role = Role::Follower; + tracing::info!( + vshard_id = self.vshard_id, + reason = why, + "calvin scheduler: this node stops leading the vShard; holding its txns" + ); + let driven: Vec = self + .pending + .iter() + .filter(|(_, pending)| leader_drives(pending.commit_state)) + .map(|(txn_id, _)| *txn_id) + .collect(); + for txn_id in driven { + self.release_to_following(txn_id); + } + // A barrier waits only on the leader. Its txn holds its locks and + // follows the log like any other. + let barriers: Vec = self.dependent_barrier.keys().copied().collect(); + for txn_id in barriers { + if let Some(barrier) = self.dependent_barrier.remove(&txn_id) { + self.follow(barrier.txn, txn_id, barrier.lock_owner); + } + } + } + + /// Hold `txn_id` `Following`, and discard what this node staged for it. + fn release_to_following(&mut self, txn_id: TxnId) { + let Some(pending) = self.pending.get_mut(&txn_id) else { + return; + }; + pending.commit_state = CommitState::Following; + pending.awaiting = None; + pending.verdict_deadline = None; + pending.stage_error = None; + pending.superseded = false; + pending.redo = None; + pending.gates.clear(); + pending.ungated = false; + self.owed.remove(&(txn_id, OwedKind::Vote)); + self.discard_staged(txn_id); + } + + /// Dispatch a `CalvinDrop` that clears what this node staged for + /// `txn_id`. An absent staged entry makes it a no-op on the core. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn discard_staged( + &mut self, + txn_id: TxnId, + ) { + if let DispatchOutcome::Failed(error) = self.dispatch_drop(txn_id, DispatchStep::Discard) { + self.fail_dispatch_step(txn_id, DispatchStep::Discard, error); + } + } + + /// Note that a redo proposal found this node no longer leads. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn on_not_leader(&mut self) { + if self.role.is_leader() { + self.demote("a redo proposal found this node no longer leads"); + } + self.role.term = None; + self.role.term_start = None; + } +} + +/// Whether a txn in `state` is one the leader drives: it holds staged state +/// or waits on a step only the leader takes. +fn leader_drives(state: CommitState) -> bool { + match state { + CommitState::Staged + | CommitState::AwaitingVerdict + | CommitState::AwaitingResolveTurn + | CommitState::AwaitingRedoResolve + | CommitState::AwaitingRedoApply { .. } => true, + // A drop ends the txn with no log entry whatever the role. + CommitState::AwaitingDrop | CommitState::Following => false, + } +} + +#[cfg(test)] +mod tests { + use nodedb_cluster::calvin::CalvinCompletionRegistry; + use nodedb_cluster::calvin::types::{ + EngineKeySet, ReadWriteSet, SchedulerInput, SequencedTxn, SortedVec, + }; + use nodedb_physical::physical_plan::PhysicalPlan; + use nodedb_physical::physical_plan::meta::MetaOp; + + use super::*; + use crate::bridge::dispatch::CoreChannelDataSide; + use crate::bridge::envelope::{StageVote, Status}; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ + elect_data_group_leader, lead_data_group, step_down_from_data_group, + }; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler_with_data_side, make_validate_only_txn, staged_response, + test_coll_vshard, + }; + + /// A scheduler on the `test_coll` vShard with its Data Plane side. + fn scheduler() -> (Scheduler, tempfile::TempDir, CoreChannelDataSide) { + build_test_scheduler_with_data_side( + test_coll_vshard(), + CalvinCompletionRegistry::new_detached(), + ) + } + + /// A validate-only txn at `(epoch, 0)` that locks `test_coll` row + /// `surrogate`, so txns on distinct rows hold their locks together. + fn validate_only_on(epoch: u64, surrogate: u32) -> SequencedTxn { + let mut txn = make_validate_only_txn(epoch, 0); + txn.tx_class.write_set = ReadWriteSet::new(vec![EngineKeySet::Document { + collection: "test_coll".to_string(), + surrogates: SortedVec::new(vec![surrogate]), + }]); + txn + } + + /// The `(epoch, position)` of every Calvin stage request on the Data + /// Plane side, in arrival order. Drains the request ring. + fn staged_requests(data_side: &mut CoreChannelDataSide) -> Vec<(u64, u32)> { + let mut staged = Vec::new(); + while let Ok(request) = data_side.request_rx.try_pop() { + if let PhysicalPlan::Meta(MetaOp::CalvinExecuteStatic { + epoch, position, .. + }) = request.inner.plan + { + staged.push((epoch, position)); + } + } + staged + } + + /// Whether a `CalvinDrop` of `txn_id` reached the Data Plane side. + /// Drains the request ring. + fn dropped(data_side: &mut CoreChannelDataSide, txn_id: TxnId) -> bool { + let mut dropped = false; + while let Ok(request) = data_side.request_rx.try_pop() { + dropped |= request.inner.plan + == PhysicalPlan::Meta(MetaOp::CalvinDrop { + epoch: txn_id.epoch, + position: txn_id.position, + }); + } + dropped + } + + /// A replica that does not lead the data group holds a granted txn + /// `Following`. It stages nothing and owes no vote. + #[tokio::test] + async fn a_follower_grant_dispatches_nothing() { + let (mut scheduler, _dir, mut data_side) = scheduler(); + let txn_id = TxnId::new(3, 0); + + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(validate_only_on(3, 1)))); + + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Following) + ); + assert!(data_side.request_rx.try_pop().is_err(), "nothing staged"); + assert!(scheduler.owed.is_empty(), "a follower owes no vote"); + assert!(!scheduler.applied.is_applied(3, 0)); + } + + /// A node that won a term stages nothing until the term's no-op applied + /// here. Once it applied, the held txn stages. + #[tokio::test] + async fn a_new_leader_stages_nothing_before_the_commit_index_applies() { + let (mut scheduler, _dir, mut data_side) = scheduler(); + let group_id = elect_data_group_leader(&scheduler); + scheduler.refresh_role(); + assert!(!scheduler.role.is_leader(), "the gate stays closed"); + assert!(scheduler.role.awaits_term_start()); + let txn_id = TxnId::new(3, 0); + + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(validate_only_on(3, 1)))); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Following) + ); + assert!(staged_requests(&mut data_side).is_empty()); + + let term_start = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .term_start_index(group_id) + .expect("the elected node names its term start"); + scheduler + .shared + .applied_index_watcher(group_id) + .bump(term_start); + scheduler.refresh_role(); + + assert!(scheduler.role.is_leader(), "the gate opens"); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Staged) + ); + assert_eq!(staged_requests(&mut data_side), vec![(3, 0)]); + } + + /// Promotion stages every `Following` txn, in sequencer order. + #[tokio::test] + async fn promotion_stages_following_txns_in_sequencer_order() { + let (mut scheduler, _dir, mut data_side) = scheduler(); + for (epoch, surrogate) in [(3, 1), (4, 2), (5, 3)] { + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(validate_only_on( + epoch, surrogate, + )))); + } + assert!(staged_requests(&mut data_side).is_empty()); + + lead_data_group(&mut scheduler); + + assert_eq!( + staged_requests(&mut data_side), + vec![(3, 0), (4, 0), (5, 0)] + ); + for epoch in [3, 4, 5] { + assert_eq!( + scheduler + .pending + .get(&TxnId::new(epoch, 0)) + .map(|p| p.commit_state), + Some(CommitState::Staged) + ); + } + } + + /// Demotion discards what the leader staged, drops its owed vote, and + /// holds the txn `Following` with its locks. + #[tokio::test] + async fn demotion_drops_staged_state_and_owed_votes() { + let (mut scheduler, _dir, mut data_side) = scheduler(); + lead_data_group(&mut scheduler); + let txn_id = TxnId::new(3, 0); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(validate_only_on(3, 1)))); + assert_eq!(staged_requests(&mut data_side), vec![(3, 0)]); + scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); + assert!(scheduler.owed.contains_key(&(txn_id, OwedKind::Vote))); + + step_down_from_data_group(&scheduler); + scheduler.refresh_role(); + + assert!(!scheduler.role.is_leader()); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Following) + ); + assert!( + !scheduler.owed.contains_key(&(txn_id, OwedKind::Vote)), + "a demoted node keeps no vote owed" + ); + assert!( + dropped(&mut data_side, txn_id), + "the staged state is dropped" + ); + assert!(!scheduler.applied.is_applied(3, 0)); + let blocked = validate_only_on(4, 1); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(blocked))); + assert!( + scheduler.blocked.contains_key(&TxnId::new(4, 0)), + "the held txn keeps its locks" + ); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs index 60892742b..b3a0767ea 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs @@ -10,6 +10,10 @@ use super::catch_up::CatchUpDrain; use super::scheduler::Scheduler; use crate::control::shutdown::ShutdownReceiver; +/// How often a new data-group leader checks whether its term's no-op +/// applied, while its stage gate is closed. +const ROLE_POLL: std::time::Duration = std::time::Duration::from_millis(5); + impl Scheduler { /// Run the scheduler event loop until shutdown is signaled. pub async fn run(mut self, mut shutdown: ShutdownReceiver) { @@ -30,6 +34,8 @@ impl Scheduler { // Woken when a routed Data-Plane response frees dispatcher capacity. let capacity_freed = Arc::clone(&self.capacity_freed); + // Woken when the data-group apply loop concludes a stamped redo. + let inbox = Arc::clone(self.inbox.inbox()); // Set when a tick left armed catch-up unreplayed. The next open-gate // pass fires the tick at once to resume it. let mut catch_up_resume = false; @@ -45,6 +51,10 @@ impl Scheduler { self.redispatch_deferred(); } + // Every pass reads the role first: a promotion stages held txns, + // a demotion stops staging. + self.refresh_role(); + self.drain_inbox(); self.check_dependent_barrier_timeouts(); self.check_awaiting_verdict_stalls(); self.resume_metadata_hold(); @@ -74,10 +84,14 @@ impl Scheduler { self.log_superseded("completion"); break; }; - // Awaited in the arm: the loop takes no other input - // until this completion, its durability wait included, - // is fully handled. - self.handle_completion(txn_id, request_id, resp_opt).await; + self.handle_completion(txn_id, request_id, resp_opt); + } + + // The apply loop concluded a stamped redo. Drained in every + // state, halted included, so the apply loop never waits on + // this scheduler. + () = inbox.ready() => { + self.drain_inbox(); } maybe_verdict = self.verdict_rx.recv() => { @@ -86,7 +100,7 @@ impl Scheduler { break; }; // A durable global verdict landed: resume the matching - // parked txn into its flush (commit) or drop (abort). + // txn. self.handle_verdict_signal(signal); } @@ -116,17 +130,16 @@ impl Scheduler { // The next loop pass re-checks the held txn's floor. } + _ = tokio::time::sleep(ROLE_POLL), if self.role.awaits_term_start() => { + // The next loop pass checks whether the term's no-op + // applied, and stages the held txns once it has. + } + _ = &mut capacity_notified, if self.resends_deferred() => { // Capacity freed: the next loop pass re-sends deferred // requests in FIFO order. } - _ = self.install_gate.released(), if self.install_gate.is_waiting() => { - // A snapshot released a group's gate: the waiting flush - // tries again. - self.pump_flush_turn(); - } - maybe_txn = self.receiver.recv(), if intake_open => { match maybe_txn { Some(input) => self.process_scheduler_input(input), @@ -150,6 +163,11 @@ impl Scheduler { !intake_open || self.drain_catch_up() == CatchUpDrain::Remaining; // Propose again every owed sequencer entry not yet applied. self.retry_owed_sequencer_entries(); + // Propose again every committed slice's redo that is not + // in the log. Apply events are drained first, so an + // install already reported is never proposed again. + self.drain_inbox(); + self.retry_redo_proposals(); // The top-of-loop check_awaiting_verdict_stalls / // check_dependent_barrier_timeouts and the deferred re-send // pass run on every wake; this arm guarantees the loop wakes @@ -157,8 +175,6 @@ impl Scheduler { } } } - // Every txn still pending stays unapplied on this replica. - self.hold_all_redo_records(); } /// Log that a scheduler channel closed and the loop exits. diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs index 9b2ea0f06..43445648e 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs @@ -49,10 +49,12 @@ pub struct Scheduler { /// Shared control-plane state used for dispatch, response tracking, WAL, /// and request-id allocation. pub(in crate::control::cluster::calvin::scheduler::driver::core) shared: Arc, - /// Handle to MultiRaft for the data-group leader check and the catch-up - /// read of the sequencer log. + /// Handle to MultiRaft for the data-group role, the redo proposals, and + /// the catch-up read of the sequencer log. pub(in crate::control::cluster::calvin::scheduler::driver::core) multi_raft: Arc>, + /// This node's role in the vShard's data group. See [`super::role`]. + pub(in crate::control::cluster::calvin::scheduler::driver::core) role: super::role::StageGate, /// Hands sequencer entries to the sequencer group, locally on its leader /// and by forward from any other node. pub(in crate::control::cluster::calvin::scheduler::driver::core) sequencer_proposer: @@ -79,11 +81,20 @@ pub struct Scheduler { /// for the brief probe the gate takes. pub(in crate::control::cluster::calvin::scheduler::driver::core) lock_manager: Arc>, - /// In-flight static/active transactions awaiting executor response, - /// including those whose request waits in `deferred` for capacity. - /// `BTreeMap` ensures deterministic iteration order. + /// Granted transactions that have not finished: staged ones on the + /// leader, held ones on a follower, including those whose request waits + /// in `deferred` for capacity. `BTreeMap` ensures deterministic + /// iteration order. pub(in crate::control::cluster::calvin::scheduler::driver::core) pending: BTreeMap, + /// Installs the apply loop reported for txns still waiting for their + /// locks. Each completes when its locks are granted. + pub(in crate::control::cluster::calvin::scheduler::driver::core) early_applied: + BTreeMap, + /// This scheduler's registered inbox: how the data-group apply loop + /// concluded each stamped redo of the vShard. + pub(in crate::control::cluster::calvin::scheduler::driver::core) inbox: + crate::control::cluster::calvin::scheduler::InboxHandle, /// Blocked transactions awaiting lock release. pub(in crate::control::cluster::calvin::scheduler::driver::core) blocked: BTreeMap, @@ -183,7 +194,7 @@ pub struct Scheduler { /// staged yet. See [`super::parts`]. pub(in crate::control::cluster::calvin::scheduler::driver::core) parts: super::parts::PartsState, - /// Orders flushes against the data group's snapshots. See + /// The vShard base this scheduler started under. See /// [`super::install_gate`]. pub(in crate::control::cluster::calvin::scheduler::driver::core) install_gate: super::install_gate::InstallGate, @@ -259,6 +270,10 @@ impl Scheduler { // A backup's cut waits on every scheduler this node runs. shared.calvin.cuts.register(vshard_id); let install_gate = super::install_gate::InstallGate::new(&shared, vshard_id); + let inbox = shared + .calvin + .inboxes + .register(vshard_id, config.max_inflight_backlog); let capacity_freed = shared .dispatcher @@ -271,11 +286,14 @@ impl Scheduler { receiver, shared, multi_raft, + role: super::role::StageGate::default(), sequencer_proposer, owed: OwedEntries::new(), sequencer_state_machine, lock_manager, pending: BTreeMap::new(), + early_applied: BTreeMap::new(), + inbox, blocked: BTreeMap::new(), dependent_barrier: BTreeMap::new(), read_result_rx, diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/staged_vote.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/staged_vote.rs index 3edf06b3d..398ab057d 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/staged_vote.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/staged_vote.rs @@ -5,7 +5,7 @@ use nodedb_cluster::calvin::AbortReason; -use crate::bridge::envelope::{Response, StageVote, Status}; +use crate::bridge::envelope::{ErrorCode, Response, StageVote, Status}; /// A participant's local verdict on its staged slice. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -57,18 +57,27 @@ pub(super) enum StageVoteError { /// A stage response that did not succeed carries a commit vote. #[error("stage returned {status:?} with a commit stage vote")] CommitWithoutSuccess { status: Status }, + /// The vShard's core fail-stopped and refused the stage. It refuses + /// every later request too, the committed install included, so no vote + /// this replica casts can be kept. + #[error("stage refused by fail-stopped core {core_id}")] + CoreFailStopped { core_id: usize }, } /// Read the local vote of a staged slice from its stage response. /// /// The response's stage vote maps 1:1. An error response with no vote never /// reached a stage path: the request expired, or its core is gone or -/// drained. That slice never staged, so it votes `ParticipantError`. +/// drained. That slice never staged, so it votes `ParticipantError`. A +/// fail-stopped core's refusal is the exception: it is refused. /// /// A response that did not fail and carries no vote is refused, and so is a /// commit vote on a response that did not succeed. Neither counts as a commit. pub(super) fn staged_commit_vote(response: &Response) -> Result { let status = response.status; + if let Some(ErrorCode::CoreFailStopped { core_id, .. }) = response.error_code.as_deref() { + return Err(StageVoteError::CoreFailStopped { core_id: *core_id }); + } match (response.stage_vote, status) { (Some(StageVote::Commit), Status::Ok) => Ok(StagedVote::Commit), (Some(StageVote::Commit), Status::Partial | Status::Error) => { @@ -90,7 +99,7 @@ pub(super) fn staged_commit_vote(response: &Response) -> Result) -> Response { @@ -184,6 +193,21 @@ mod tests { ); } + /// A fail-stopped core's refusal is no vote: the core refuses the + /// committed install too. + #[test] + fn a_fail_stopped_core_refusal_is_refused() { + let mut response = staged_response(Status::Error, None); + response.error_code = Some(Box::new(ErrorCode::CoreFailStopped { + core_id: 2, + detail: "rollback failed".into(), + })); + assert_eq!( + staged_commit_vote(&response), + Err(StageVoteError::CoreFailStopped { core_id: 2 }) + ); + } + #[test] fn only_an_abort_carries_a_reason() { assert_eq!(StagedVote::Commit.abort_reason(), None); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_proposer.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_proposer.rs index 6602ec96e..b08ede769 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_proposer.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_proposer.rs @@ -72,9 +72,28 @@ impl SequencerProposer for CapturingProposer { } /// Make the fixture node the elected leader of the data group that owns -/// `scheduler`'s vShard, so leader-only proposals such as the commit vote -/// run. -pub(super) fn elect_data_group_leader(scheduler: &Scheduler) { +/// `scheduler`'s vShard, apply the term's no-op, and open the scheduler's +/// stage gate, so it stages its grants and proposes its entries. +pub(super) fn lead_data_group(scheduler: &mut Scheduler) { + let group_id = elect_data_group_leader(scheduler); + let term_start = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .term_start_index(group_id) + .expect("the elected node names its term start"); + scheduler + .shared + .applied_index_watcher(group_id) + .bump(term_start); + scheduler.refresh_role(); + assert!(scheduler.role.is_leader(), "the stage gate opens"); +} + +/// Make the fixture node the elected leader of the data group that owns +/// `scheduler`'s vShard, with its term's no-op not yet applied. Returns +/// the group. +pub(super) fn elect_data_group_leader(scheduler: &Scheduler) -> u64 { let mut mr = scheduler .multi_raft .lock() @@ -95,8 +114,38 @@ pub(super) fn elect_data_group_leader(scheduler: &Scheduler) { for _ in 0..20 { mr.tick().expect("tick"); if mr.vshard_role_is_leader(scheduler.vshard_id) { - return; + return group_id; } } panic!("data group did not elect the single fixture node"); } + +/// Step the fixture node down from its data group: a new term starts on +/// another node. +pub(super) fn step_down_from_data_group(scheduler: &Scheduler) { + let mut mr = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()); + let group_id = mr + .routing() + .read() + .unwrap_or_else(|p| p.into_inner()) + .group_for_vshard(scheduler.vshard_id) + .expect("the fixture routing maps every vShard"); + let term = mr.leader_term(group_id).expect("the fixture node leads"); + // A heartbeat of a later term from another node deposes this one. + mr.handle_append_entries(&nodedb_raft::message::AppendEntriesRequest { + term: term + 1, + leader_id: scheduler.shared.node_id + 1, + prev_log_index: 0, + prev_log_term: 0, + entries: Vec::new(), + leader_commit: 0, + group_id, + round: 0, + replicated_floor: 0, + }) + .expect("the group takes the heartbeat"); + assert!(!mr.vshard_role_is_leader(scheduler.vshard_id)); +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs index ab7e7726b..891f3b26d 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs @@ -28,7 +28,9 @@ use crate::control::cluster::calvin::scheduler::driver::core::scheduler::{ Scheduler, SchedulerParams, }; use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::CapturingProposer; -use crate::control::cluster::calvin::scheduler::driver::types::{CommitState, PendingTxn}; +use crate::control::cluster::calvin::scheduler::driver::types::{ + CommitState, PendingTxn, SliceScope, +}; use crate::control::cluster::calvin::scheduler::lock_manager::{LockManager, TxnId}; use crate::control::cluster::calvin::scheduler::metrics::SchedulerMetrics; use crate::control::shutdown::ShutdownWatch; @@ -428,26 +430,49 @@ pub(super) fn spawn_scheduler_loop(mut scheduler: Scheduler) -> RunningScheduler } } -/// A `PendingTxn` staged and parked awaiting the cross-shard commit verdict. +/// A `PendingTxn` staged on the leader, its stage answer awaited. Its slice +/// writes `test_coll`. pub(super) fn staged_pending(txn: SequencedTxn, txn_id: TxnId) -> PendingTxn { + let mut scope = SliceScope::of_plans(&[]); + scope.writes = true; + scope.collections = vec!["test_coll".to_string()]; PendingTxn { txn, lock_owner: txn_id, // no-determinism: test-only dispatch timestamp for a fabricated PendingTxn fixture. dispatch_time: Instant::now(), has_primary_write: true, + raises_write_mark: true, has_returning: false, - change_sets: Vec::new(), commit_state: CommitState::Staged, + awaiting: None, verdict_deadline: None, stage_error: None, - redo_records: None, - flush_scope: crate::control::cluster::calvin::scheduler::driver::types::FlushScope::default( - ), + scope, + redo: None, superseded: false, gates: Vec::new(), ungated: false, - install_permit: None, + } +} + +/// A `PendingTxn` a follower holds for its verdict and its redo. +pub(super) fn following_pending(txn: SequencedTxn, txn_id: TxnId) -> PendingTxn { + let mut pending = staged_pending(txn, txn_id); + pending.commit_state = CommitState::Following; + pending.has_primary_write = false; + pending.raises_write_mark = false; + pending +} + +/// The answer the apply loop reports for an installed slice. +pub(super) fn redo_applied( + reply: Response, +) -> crate::control::cluster::calvin::scheduler::CalvinApplyEvent { + crate::control::cluster::calvin::scheduler::CalvinApplyEvent::RedoApplied { + reply, + primary_write: true, + returning: false, } } @@ -484,7 +509,8 @@ pub(super) fn begin_data_plane_drain(shared: &SharedState) { .begin_data_plane_drain(); } -/// A scheduler on vShard 7 with `txn_id` pending in `state`. +/// A scheduler on vShard 7 with `txn_id` pending in `state`, its answer to +/// request 9 awaited. pub(super) fn scheduler_with_pending( txn_id: TxnId, state: CommitState, @@ -492,6 +518,7 @@ pub(super) fn scheduler_with_pending( let (mut scheduler, dir) = build_test_scheduler(7); let mut pending = staged_pending(make_sequenced_txn(txn_id.epoch, txn_id.position), txn_id); pending.commit_state = state; + pending.awaiting = Some(RequestId::new(9)); scheduler.pending.insert(txn_id, pending); (scheduler, dir) } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/write_version_record.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/write_version_record.rs deleted file mode 100644 index 2a68d70c2..000000000 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/write_version_record.rs +++ /dev/null @@ -1,306 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Post-apply write-version recording for committed Calvin transactions. -//! -//! A Calvin apply's committed WAL LSN is the LSN of its `TransactionRedo` -//! record, or of the `CalvinApplied` marker a transaction that wrote nothing -//! here appends. The install records the collection floors and index-value -//! versions of the record at its LSN. The per-key versions of the local write -//! plans are recorded here: the scheduler dispatches a one-way, record-only op -//! back to the same core, which funnels the plans through the shared -//! write-version recorder at that LSN — the same shard-local WAL-LSN space -//! the single-shard fast path and read watermarks use. - -use super::super::types::CommitState; -use super::deferred::{DispatchOutcome, DispatchStep}; -use super::halt::{HaltReason, HaltStep}; -use super::scheduler::Scheduler; -use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; -use crate::types::Lsn; -use nodedb_physical::physical_plan::PhysicalPlan; -use nodedb_physical::physical_plan::meta::MetaOp; - -/// Result of [`Scheduler::record_calvin_write_versions`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(in crate::control::cluster::calvin::scheduler::driver::core) enum VersionRecord { - /// The dispatcher accepted the record. - Sent, - /// The record waits for capacity. The txn stays pending until it is sent. - Parked, - /// The record cannot be built or sent. The scheduler halted, and the txn - /// stays pending and unapplied. - Halted, -} - -impl Scheduler { - /// Record the per-key write versions of a just-committed Calvin - /// transaction's locally-applied write plans at its committed WAL - /// `applied_lsn`. - /// - /// Dispatches a record-only [`MetaOp::RecordCalvinWriteVersions`] op back to - /// this vShard's core with `applied_lsn` stamped on the request envelope's - /// `wal_lsn`; the core funnels the plans through the shared write-version - /// recorder at that LSN. The recorded version therefore lands in the same - /// WAL-LSN space as fast-path writes. - /// - /// The record feeds read-set validation. A later txn's stage checks each - /// read against these versions. A missing version reports a stale read as - /// valid. So the txn keeps its locks until the record is sent. The core - /// applies requests in dispatch order, so the record lands before the - /// stage of any txn that waits on these locks. The response is drained - /// and discarded. - /// - /// A record refused at capacity is parked and re-sent once capacity - /// frees. A decode or routing error, or a terminal dispatch refusal, - /// halts the scheduler. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn record_calvin_write_versions( - &mut self, - txn_id: TxnId, - applied_lsn: Lsn, - ) -> VersionRecord { - let epoch = txn_id.epoch; - let position = txn_id.position; - - let Some(pending) = self.pending.get(&txn_id) else { - return self.halt_version_record(txn_id, "the txn has no pending entry".to_string()); - }; - let tenant_id = pending.txn.tx_class.tenant_id; - let database_id = pending.txn.tx_class.database_id; - let plans = match super::super::helpers::decode_plans(&pending.txn.tx_class.plans) { - Ok(p) => p, - Err(e) => return self.halt_version_record(txn_id, format!("plan decode: {e}")), - }; - // The locally-applied slice this vShard committed. Empty for a - // validate-only READ participant (no local writes) — the recorder then - // has nothing to record, which is correct. The recorder no-ops any plan - // without a per-key or collection version, so no gate on plan kind is - // applied here — gating on a narrower write predicate would silently - // skip recordable writes (e.g. a CRDT apply) and reopen the version gap. - let local = match self.local_calvin_plans(plans, database_id, epoch, position) { - Ok(p) => p, - Err(e) => { - return self.halt_version_record(txn_id, format!("local plan routing: {e}")); - } - }; - - let request_id = self.next_request_id(); - let plan = PhysicalPlan::Meta(MetaOp::RecordCalvinWriteVersions { - tenant_id, - plans: local, - }); - // The committed write-LSN for this Calvin apply — recorded against - // every key the plans wrote, in the same WAL-LSN space as fast-path. - let request = - self.build_exempt_request(request_id, tenant_id, database_id, plan, Some(applied_lsn)); - - match self.dispatch_sequenced(txn_id, DispatchStep::WriteVersionRecord, request) { - DispatchOutcome::Sent => VersionRecord::Sent, - DispatchOutcome::Deferred => VersionRecord::Parked, - DispatchOutcome::Failed(error) => { - self.fail_dispatch_step(txn_id, DispatchStep::WriteVersionRecord, error); - VersionRecord::Halted - } - } - } - - /// Complete `txn_id` once its parked write-version record is sent, when - /// its commit tail already ran. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn complete_after_version_record( - &mut self, - txn_id: TxnId, - ) { - let held = self - .pending - .get(&txn_id) - .is_some_and(|pending| pending.commit_state == CommitState::AwaitingVersionRecord); - if held { - self.metrics.record_completed(); - self.on_txn_complete(txn_id); - } - } - - /// Halt on a write-version record that cannot be built. - fn halt_version_record(&mut self, txn_id: TxnId, error: String) -> VersionRecord { - self.halt_apply( - txn_id, - HaltReason::WriteVersionRecordFailed, - HaltStep::WriteVersionRecord, - error, - ); - VersionRecord::Halted - } -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use nodedb_cluster::calvin::CalvinCompletionRegistry; - use nodedb_types::TenantId; - - use super::*; - use crate::control::cluster::calvin::scheduler::driver::core::intake::IntakeClosure; - use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ - await_data_plane_request, begin_data_plane_drain, build_test_scheduler, - build_test_scheduler_with_data_side, fill_tenant_inflight, make_validate_only_txn, - release_filler, spawn_scheduler_loop, staged_pending, test_coll_vshard, - }; - - /// Once a Data Plane response frees tenant capacity, a write-version - /// record refused at capacity reaches the Data Plane. - #[tokio::test] - async fn refused_write_version_record_reaches_data_plane_after_capacity_frees() { - let registry = CalvinCompletionRegistry::new_detached(); - let (mut scheduler, _dir, mut data_side) = - build_test_scheduler_with_data_side(test_coll_vshard(), registry); - let txn_id = TxnId::new(21, 0); - scheduler.pending.insert( - txn_id, - staged_pending(make_validate_only_txn(21, 0), txn_id), - ); - let shared = Arc::clone(&scheduler.shared); - let fillers = fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); - - assert_eq!( - scheduler.record_calvin_write_versions(txn_id, Lsn::new(42)), - VersionRecord::Parked - ); - let running = spawn_scheduler_loop(scheduler); - release_filler(&shared, &mut data_side, fillers[0]); - - let arrived = await_data_plane_request(&mut data_side, |plan| { - matches!( - plan, - PhysicalPlan::Meta(MetaOp::RecordCalvinWriteVersions { .. }) - ) - }) - .await; - running.stop().await; - - assert!( - arrived, - "the refused write-version record must reach the Data Plane once capacity frees" - ); - } - - /// A write-version record refused terminally halts the scheduler. The - /// txn stays pending and unapplied, and intake closes. - #[tokio::test] - async fn terminal_write_version_record_refusal_halts_and_holds_the_txn() { - let registry = CalvinCompletionRegistry::new_detached(); - let (mut scheduler, _dir, _data_side) = - build_test_scheduler_with_data_side(test_coll_vshard(), registry); - let txn_id = TxnId::new(22, 0); - scheduler.pending.insert( - txn_id, - staged_pending(make_validate_only_txn(22, 0), txn_id), - ); - begin_data_plane_drain(&scheduler.shared); - - assert_eq!( - scheduler.record_calvin_write_versions(txn_id, Lsn::new(42)), - VersionRecord::Halted - ); - - assert!(!scheduler.applied.is_applied(22, 0)); - assert!(scheduler.pending.contains_key(&txn_id)); - assert_eq!( - scheduler.apply_halt().map(|h| h.report.step), - Some("write_version_record") - ); - assert_eq!(scheduler.intake_closure(), Some(IntakeClosure::ApplyHalted)); - } - - /// A parked write-version record refused terminally on re-send halts the - /// scheduler, and its held txn stays pending and unapplied. - #[tokio::test] - async fn parked_write_version_record_refused_on_resend_holds_the_txn() { - let registry = CalvinCompletionRegistry::new_detached(); - let (mut scheduler, _dir, mut data_side) = - build_test_scheduler_with_data_side(test_coll_vshard(), registry); - let txn_id = TxnId::new(23, 0); - scheduler.pending.insert( - txn_id, - staged_pending(make_validate_only_txn(23, 0), txn_id), - ); - let shared = Arc::clone(&scheduler.shared); - fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); - assert_eq!( - scheduler.record_calvin_write_versions(txn_id, Lsn::new(42)), - VersionRecord::Parked - ); - if let Some(pending) = scheduler.pending.get_mut(&txn_id) { - pending.commit_state = CommitState::AwaitingVersionRecord; - } - - begin_data_plane_drain(&shared); - scheduler.redispatch_deferred(); - - assert!(!scheduler.applied.is_applied(23, 0)); - assert!(scheduler.pending.contains_key(&txn_id)); - assert_eq!( - scheduler.apply_halt().map(|h| h.report.step), - Some("write_version_record") - ); - assert!(!scheduler.resends_deferred()); - } - - /// A txn held for its parked write-version record completes once the - /// record is sent. - #[tokio::test] - async fn held_txn_completes_once_its_write_version_record_is_sent() { - let registry = CalvinCompletionRegistry::new_detached(); - let (mut scheduler, _dir, mut data_side) = - build_test_scheduler_with_data_side(test_coll_vshard(), registry); - let txn_id = TxnId::new(24, 0); - scheduler.pending.insert( - txn_id, - staged_pending(make_validate_only_txn(24, 0), txn_id), - ); - let shared = Arc::clone(&scheduler.shared); - let fillers = fill_tenant_inflight(&shared, &mut data_side, TenantId::new(1)); - assert_eq!( - scheduler.record_calvin_write_versions(txn_id, Lsn::new(42)), - VersionRecord::Parked - ); - if let Some(pending) = scheduler.pending.get_mut(&txn_id) { - pending.commit_state = CommitState::AwaitingVersionRecord; - } - assert!( - !scheduler.applied.is_applied(24, 0), - "a parked record holds the position unapplied" - ); - - release_filler(&shared, &mut data_side, fillers[0]); - scheduler.redispatch_deferred(); - - assert!(!scheduler.has_deferred_dispatch()); - assert!(!scheduler.pending.contains_key(&txn_id)); - assert!(scheduler.applied.is_applied(24, 0)); - assert!(!scheduler.is_apply_halted()); - } - - /// Plans that do not decode leave no record to send. The scheduler - /// halts, and the txn stays pending and unapplied. - #[tokio::test] - async fn undecodable_plans_halt_the_write_version_record() { - let (mut scheduler, _dir) = build_test_scheduler(test_coll_vshard()); - let txn_id = TxnId::new(25, 0); - let mut pending = staged_pending(make_validate_only_txn(25, 0), txn_id); - // 0xc1 is a reserved MessagePack marker, so no plan batch decodes. - pending.txn.tx_class.plans = vec![0xc1]; - scheduler.pending.insert(txn_id, pending); - - assert_eq!( - scheduler.record_calvin_write_versions(txn_id, Lsn::new(42)), - VersionRecord::Halted - ); - - assert!(!scheduler.applied.is_applied(25, 0)); - assert!(scheduler.pending.contains_key(&txn_id)); - assert_eq!( - scheduler.apply_halt().map(|h| h.reason), - Some(HaltReason::WriteVersionRecordFailed) - ); - } -} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs index e3f237f36..3ecc4d098 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs @@ -9,15 +9,18 @@ use nodedb_cluster::calvin::types::SequencedTxn; use super::super::lock_manager::{LockKey, LockMode, TxnId}; -/// An in-flight transaction that has been dispatched and is awaiting a -/// Data Plane response. +/// A transaction this vShard granted and has not finished. +/// +/// On the data-group leader it stages, votes, resolves and proposes its +/// redo. On a follower it waits in [`CommitState::Following`] for its +/// verdict and its stamped redo. /// /// The executor response channel is held by a bridge task (see /// `Scheduler::spawn_response_bridge`) that forwards completions to the /// scheduler's fan-in `completion_rx`. This avoids polling and ensures the /// main `select!` loop wakes the moment a response arrives. pub(super) struct PendingTxn { - /// Original sequenced transaction (for WAL record on completion). + /// Original sequenced transaction. pub txn: SequencedTxn, /// The lock-table owner id for this txn (equals the apply-slot id unless a /// reservation owns the lock). Used by `on_txn_complete` to `release` the @@ -36,83 +39,68 @@ pub(super) struct PendingTxn { /// cleanup participants that dual-home alongside it carry no primary write and /// so never clobber the entry the coordinator drains. pub has_primary_write: bool, + /// Whether the slice's install raises its tenant's write mark: a + /// primary write that changes a tenant row. A schema install raises none. + pub raises_write_mark: bool, /// Whether this vShard's slice carries a RETURNING-bearing write (a plan - /// whose applied response is DATA-ROWs, not a bare affected-count). Used at - /// the commit tail to detect a genuine cross-shard RETURNING union: two - /// returning-bearing participants for one txn are unsupported, whereas - /// multiple plain-write participants (a multi-collection cross-shard COMMIT) - /// coalesce without conflict. + /// whose applied response is DATA-ROWs, not a bare affected-count). Two + /// returning-bearing participants for one txn are an unsupported + /// cross-shard RETURNING union. pub has_returning: bool, - /// Participant-local Control-Plane change manifests. Consumed once after - /// durable COMMIT apply; graph dual-home replicas are excluded at capture. - pub change_sets: Vec, /// Commit-resolution state of the txn. - /// - /// Every dispatch stages the txn and starts in [`CommitState::Staged`]. - /// The first executor response carries the local commit vote. It drives a - /// flush-or-drop before the commit tail runs. pub commit_state: CommitState, + /// The request whose response the txn waits for. A response to any + /// other request of the txn is from a step it left, and is ignored. + pub awaiting: Option, /// Stall deadline for a txn parked in [`CommitState::AwaitingVerdict`]. /// /// `Some(instant)` only while parked: if the deadline passes with the /// durable global verdict still unknown, the scheduler emits a stall /// warning and re-arms this deadline — it NEVER releases locks and NEVER - /// aborts (a unilateral abort while a peer can have already flushed a commit + /// aborts (a unilateral abort while a peer can have already committed /// will tear the transaction). `None` in every other state. /// /// `Instant::now()` is used for this deadline (observability / liveness /// only; never influences WAL bytes). pub verdict_deadline: Option, - /// Error text of a stage response that was not `Ok` on this replica. + /// Error text of a stage response that was not `Ok` on this leader. /// - /// `Some` when this replica never staged the txn. An abort verdict drops it - /// as usual. A COMMIT verdict halts the scheduler, because the txn cannot - /// apply here while its peers apply it. + /// `Some` when this leader never staged the txn. An abort verdict drops + /// it as usual. A COMMIT verdict halts the scheduler, because the txn + /// cannot apply here while its peers apply it. pub stage_error: Option, - /// The `TransactionRedo` record appended for a committed txn's flush, - /// under its outcome-floor window. The window settles when the txn - /// completes, and holds when the scheduler halts or stops with the txn - /// still pending. - pub redo_records: Option, - /// What a committed flush names beside its redo record, derived once - /// from this vShard's slice when it stages. - pub flush_scope: FlushScope, + /// What this vShard's slice writes, derived once from its local plans. + pub scope: SliceScope, + /// The committed slice's redo, kept while it is proposed so a stall or + /// a new term can propose it again. + pub redo: Option, /// A collection the transaction names no longer held its planned - /// incarnation when this replica staged it. The vote is an abort. + /// incarnation when this leader staged it. The vote is an abort. pub superseded: bool, /// The gates of the collections the transaction names, held shared from /// the incarnation check until the txn leaves `pending`. A purge of one - /// waits, so a staged slice never flushes into a recreated collection. + /// waits, so a staged slice never commits into a recreated collection. pub gates: Vec, - /// A halt released `gates` while the txn had no write in flight. Its - /// flush checks the incarnations again and takes the gates back first. + /// A halt released `gates` while the txn waited. Its proposal checks the + /// incarnations again and takes the gates back first. pub ungated: bool, - /// The shared hold of the vShard's data-group apply gate, taken when the - /// flush dispatches and dropped once the txn left `pending` and its - /// position is marked applied. A data-group snapshot capture or install - /// holds the gate exclusive, so it never sees a flush half applied. - pub install_permit: Option, } -/// What a vShard's committed flush carries: the collections its slice -/// writes, their materialized-sum targets, and the redo record. The Data -/// Plane installs the record the way every committed transaction installs. -/// -/// The collections and sum targets are derived when the slice stages, from -/// the plans the stage sends. A flush that follows a COMMIT verdict -/// therefore never decodes or routes a plan, so it cannot fail after the -/// verdict is durable. -#[derive(Debug, Clone, Default)] -pub(super) struct FlushScope { +/// What this vShard's slice of a transaction writes: derived once from its +/// local plans, on every replica alike. +#[derive(Debug, Clone)] +pub(super) struct SliceScope { + /// Whether the slice holds any local write plan. A slice with none only + /// validates reads: it commits with no log entry. + pub writes: bool, pub collections: Vec, pub sum_targets: Vec, - /// The encoded redo record the resolve appended. Empty until the resolve - /// answers, and empty when the slice wrote nothing on this vShard. A - /// resent flush carries the same bytes. - pub redo: Vec, - /// Flushes sent for this txn. A refused install resends the flush until - /// the count reaches its bound. - pub sends: u32, + /// The identities the local plans carry, bound in this node's catalog. + /// Every replica binds them before the redo installs. + pub identities: Vec, + /// Which commit-boundary checks the install runs: a RESTORE's, or a + /// commit's. + pub origin: nodedb_physical::physical_plan::RedoOrigin, /// Every plan of this vShard's slice is one a trigger body buffered, and /// a client plan of the transaction homes on another vShard. The slice /// then commits under `Trigger`: a single overlay holding the whole @@ -120,16 +108,16 @@ pub(super) struct FlushScope { pub body_only: bool, /// The slice reads a whole collection at resolve: a TRUNCATE of rows. A /// TRUNCATE's edge share reads none. A resolve otherwise runs as soon as - /// its verdict arrives, - /// beside lower txns that have not flushed. A TRUNCATE's resolve then - /// misses their rows, so it waits for its flush turn. + /// its verdict arrives, beside lower txns that have not applied. A + /// TRUNCATE's resolve then misses their rows, so it waits for its turn. pub resolve_at_turn: bool, } -impl FlushScope { - /// The flush scope of the local plans `plans`. +impl SliceScope { + /// The scope of the local plans `plans`. pub(super) fn of_plans(plans: &[nodedb_physical::physical_plan::PhysicalPlan]) -> Self { Self { + writes: !plans.is_empty(), collections: crate::control::wal_replication::transaction_redo::collections::written_collections( plans, @@ -138,20 +126,38 @@ impl FlushScope { crate::control::wal_replication::transaction_redo::sum_targets::redo_sum_targets( plans, ), - // The resolve fills these once it answers. - redo: Vec::new(), - sends: 0, + // The leader fills them once it binds the plans. + identities: Vec::new(), + origin: redo_origin(plans), body_only: false, resolve_at_turn: plans.iter().any(reads_whole_collection_at_resolve), } } } +/// The origin of a Calvin slice's redo record: a RESTORE when its local +/// plans install a RESTORE batch, a commit otherwise. A RESTORE's rows +/// passed their collection's rules when they were first written, so the +/// install runs only the checks a RESTORE runs. +fn redo_origin( + plans: &[nodedb_physical::physical_plan::PhysicalPlan], +) -> nodedb_physical::physical_plan::RedoOrigin { + use nodedb_physical::physical_plan::{MetaOp, PhysicalPlan, RedoOrigin}; + if plans + .iter() + .any(|plan| matches!(plan, PhysicalPlan::Meta(MetaOp::RestoreRedo(_)))) + { + RedoOrigin::Restore + } else { + RedoOrigin::Commit + } +} + /// Whether the resolve of `plan` reads a whole collection's stored state. /// /// A TRUNCATE's edge share is not among them. Its resolve records a cut at /// the transaction's ordinal and reads no stored edge, and the cut hides by -/// ordinal whatever this vShard flushes before or after it. The rows' +/// ordinal whatever this vShard installs before or after it. The rows' /// truncate on the collection's vShard still reads every row at resolve. fn reads_whole_collection_at_resolve(plan: &nodedb_physical::physical_plan::PhysicalPlan) -> bool { use nodedb_physical::physical_plan::{ @@ -167,55 +173,44 @@ fn reads_whole_collection_at_resolve(plan: &nodedb_physical::physical_plan::Phys ) } -/// Commit-resolution state of a staged static Calvin transaction. +/// Commit-resolution state of a granted Calvin transaction. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(in crate::control::cluster::calvin::scheduler::driver) enum CommitState { + /// A follower holds the txn's locks and stages nothing. It completes on + /// an abort verdict, on a COMMIT verdict for a slice with no write, or + /// once the slice's stamped redo applies. A promotion stages it. + Following, /// Awaiting the validate-and-stage response, whose `stage_vote` carries - /// the local commit vote that drives the flush-or-drop decision. + /// the local commit vote. Staged, /// The staged txn has cast its local vote and PARKED, awaiting the durable /// authoritative GLOBAL verdict aggregated across all participant vShards. /// /// The scheduler holds this txn's locks and its staged Data-Plane buffer - /// intact while parked; it resumes into a flush (verdict = commit) or a drop - /// (verdict = abort) only once `registry.verdict(txn)` is known — via the - /// verdict push, the probe on park, or the stall re-probe sweep. This is the - /// cross-shard commit barrier: no participant self-decides on its local - /// vote, so a torn commit (one shard flushes while a peer drops) is + /// intact while parked; it resumes into a resolve (verdict = commit) or a + /// drop (verdict = abort) only once `registry.verdict(txn)` is known — via + /// the verdict push, the probe on park, or the stall re-probe sweep. No + /// participant self-decides on its local vote, so a torn commit is /// impossible. AwaitingVerdict, - /// The txn committed and a `MetaOp::CalvinResolve` has been dispatched, or - /// parked for re-send at dispatcher capacity, to resolve its staged - /// post-images into a replayable `RedoRecord`; awaiting that response - /// before the redo is WAL-appended and the flush dispatched. - AwaitingRedoResolve, /// The txn committed, and its slice reads a whole collection at resolve /// (a TRUNCATE). Its resolve waits for its turn: every lower txn of this /// vShard finished, so the resolve reads every write sequenced before it - /// and none after. See [`FlushScope::resolve_at_turn`]. + /// and none after. See [`SliceScope::resolve_at_turn`]. AwaitingResolveTurn, - /// The txn committed and its redo record is appended. Its flush waits for - /// its turn: every lower txn of this vShard finished. See - /// [`super::core::flush_turn`]. - AwaitingFlushTurn { redo_lsn: Option }, - /// A flush (`committed = true`) or drop (`committed = false`) has been - /// dispatched, or parked for re-send at dispatcher capacity; awaiting its - /// response before the commit tail runs. - /// - /// `redo_lsn` is `Some(lsn)` when a `TransactionRedo` record was appended - /// for this commit's non-empty write set — `commit_apply_tail` then only - /// records write versions at it, since the redo record already IS the - /// applied marker. `None` for a drop, or a committed txn whose resolved - /// redo carried no ops (pure read / CRDT) — `commit_apply_tail` falls back - /// to appending a `CalvinApplied` marker in that case. - AwaitingResolve { - committed: bool, - redo_lsn: Option, - }, - /// The commit tail ran, and the write-version record waits for capacity. - /// The txn completes once the record is sent. Its locks hold until then, - /// so no later txn validates a read against the missing versions. - AwaitingVersionRecord, + /// The txn committed and a `MetaOp::CalvinResolve` has been dispatched, or + /// parked for re-send at dispatcher capacity, to resolve its staged + /// post-images into a replayable `RedoRecord`. + AwaitingRedoResolve, + /// The txn committed and its stamped redo is proposed to the vShard's + /// data group. `proposed` names the `(group_id, log_index)` of the entry + /// that installs it, `None` while no proposal is in the log. The txn + /// completes once its redo applies. + AwaitingRedoApply { proposed: Option<(u64, u64)> }, + /// A `CalvinDrop` has been dispatched, or parked for re-send: the abort + /// verdict, or a COMMIT verdict for a slice with no write. The txn + /// completes once the drop answers. + AwaitingDrop, } /// A transaction that is blocked on lock acquisition. diff --git a/nodedb/src/control/cluster/calvin/scheduler/inbox.rs b/nodedb/src/control/cluster/calvin/scheduler/inbox.rs new file mode 100644 index 000000000..63356b027 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/inbox.rs @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! What the data-group apply loop tells a vShard's scheduler about each +//! stamped Calvin redo it concluded. +//! +//! The apply loop pushes one [`CalvinApplyEvent`] per position, keyed by +//! `(epoch, position)`. The scheduler drains every event in key order. The +//! inbox is bounded: a push into a full inbox waits for room, and its entry +//! holds its own place in the group's lane meanwhile. Nothing is dropped. +//! +//! A vShard with no running scheduler has no registered inbox. The apply +//! loop then pushes nothing: the ledger it marks seeds the next scheduler. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::{Arc, Mutex}; + +use tokio::sync::Notify; + +use crate::bridge::envelope::Response; + +/// How the apply loop concluded one stamped Calvin redo. +#[derive(Debug, Clone)] +pub enum CalvinApplyEvent { + /// The install is durable and the ledger holds the position. `reply` is + /// the slice's rendered answer. Every replica renders the same one. + RedoApplied { + reply: Response, + /// The slice writes a row the statement names. + primary_write: bool, + /// The slice answers rows, not a count. + returning: bool, + }, + /// The install refused for good. Every replica refuses the same entry. + /// The position stays claimed, so no later copy installs. + RedoRefused { error: String }, + /// The install did not become durable on this replica. The claim is + /// released, and a restart applies the entry again. + RedoNotApplied { error: String }, +} + +#[derive(Debug, Default)] +struct InboxState { + events: BTreeMap<(u64, u32), CalvinApplyEvent>, + /// The scheduler stopped: a push returns at once. + closed: bool, +} + +/// One vShard's inbox of apply events. +#[derive(Debug)] +pub struct CalvinVShardInbox { + state: Mutex, + /// Wakes the scheduler: an event arrived. + ready: Notify, + /// Wakes a waiting push: the scheduler took events. + room: Notify, + capacity: usize, +} + +impl CalvinVShardInbox { + /// An empty inbox holding at most `capacity` events. A zero capacity + /// holds one. + pub fn new(capacity: usize) -> Self { + Self { + state: Mutex::new(InboxState::default()), + ready: Notify::new(), + room: Notify::new(), + capacity: capacity.max(1), + } + } + + /// Hand `event` for `(epoch, position)` to the scheduler. Waits while + /// the inbox is full. Returns once the event is in, or at once when the + /// scheduler stopped. + pub async fn push(&self, epoch: u64, position: u32, event: CalvinApplyEvent) { + let mut event = Some(event); + loop { + let room = self.room.notified(); + tokio::pin!(room); + room.as_mut().enable(); + { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + if state.closed { + return; + } + let key = (epoch, position); + if state.events.len() < self.capacity || state.events.contains_key(&key) { + if let Some(event) = event.take() { + state.events.insert(key, event); + } + drop(state); + self.ready.notify_one(); + return; + } + } + room.await; + } + } + + /// Take every waiting event, in `(epoch, position)` order. + pub fn take_all(&self) -> Vec<((u64, u32), CalvinApplyEvent)> { + let taken = { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + std::mem::take(&mut state.events) + }; + if !taken.is_empty() { + self.room.notify_waiters(); + } + taken.into_iter().collect() + } + + /// Resolves once an event waits. + pub async fn ready(&self) { + loop { + let ready = self.ready.notified(); + tokio::pin!(ready); + ready.as_mut().enable(); + if !self + .state + .lock() + .unwrap_or_else(|p| p.into_inner()) + .events + .is_empty() + { + return; + } + ready.await; + } + } + + /// The scheduler stopped: every waiting push and every later push + /// returns at once. + fn close(&self) { + self.state.lock().unwrap_or_else(|p| p.into_inner()).closed = true; + self.room.notify_waiters(); + } + + /// How many events wait. + #[cfg(test)] + pub fn waiting(&self) -> usize { + self.state + .lock() + .unwrap_or_else(|p| p.into_inner()) + .events + .len() + } +} + +type Inboxes = Arc>>>; + +/// The inbox of every vShard whose scheduler runs on this node. +#[derive(Debug, Default)] +pub struct CalvinInboxes { + by_vshard: Inboxes, +} + +impl CalvinInboxes { + /// Register the inbox of the scheduler starting for `vshard_id`. It + /// replaces a predecessor's. + pub fn register(&self, vshard_id: u32, capacity: usize) -> InboxHandle { + let inbox = Arc::new(CalvinVShardInbox::new(capacity)); + let replaced = self + .by_vshard + .lock() + .unwrap_or_else(|p| p.into_inner()) + .insert(vshard_id, Arc::clone(&inbox)); + if let Some(previous) = replaced { + previous.close(); + } + InboxHandle { + vshard_id, + inbox, + by_vshard: Arc::clone(&self.by_vshard), + } + } + + /// The inbox of `vshard_id`'s running scheduler. + pub fn get(&self, vshard_id: u32) -> Option> { + self.by_vshard + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&vshard_id) + .cloned() + } +} + +/// A scheduler's registered inbox. Dropping it closes the inbox and +/// removes it, unless a successor replaced it. +#[derive(Debug)] +pub struct InboxHandle { + vshard_id: u32, + inbox: Arc, + by_vshard: Inboxes, +} + +impl InboxHandle { + /// The inbox this handle registered. + pub fn inbox(&self) -> &Arc { + &self.inbox + } +} + +impl Drop for InboxHandle { + fn drop(&mut self) { + self.inbox.close(); + let mut by_vshard = self.by_vshard.lock().unwrap_or_else(|p| p.into_inner()); + if by_vshard + .get(&self.vshard_id) + .is_some_and(|inbox| Arc::ptr_eq(inbox, &self.inbox)) + { + by_vshard.remove(&self.vshard_id); + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + fn refused(text: &str) -> CalvinApplyEvent { + CalvinApplyEvent::RedoRefused { + error: text.to_owned(), + } + } + + /// Events leave the inbox in `(epoch, position)` order, whatever order + /// they arrived in. + #[tokio::test] + async fn events_drain_in_position_order() { + let inbox = CalvinVShardInbox::new(8); + inbox.push(4, 1, refused("b")).await; + inbox.push(3, 7, refused("a")).await; + inbox.push(4, 0, refused("c")).await; + let keys: Vec<(u64, u32)> = inbox.take_all().into_iter().map(|(k, _)| k).collect(); + assert_eq!(keys, vec![(3, 7), (4, 0), (4, 1)]); + assert_eq!(inbox.waiting(), 0); + } + + /// A push into a full inbox waits until the scheduler takes events. + #[tokio::test] + async fn a_full_inbox_holds_the_push_until_events_leave() { + let inbox = Arc::new(CalvinVShardInbox::new(1)); + inbox.push(1, 0, refused("first")).await; + let pushing = { + let inbox = Arc::clone(&inbox); + tokio::spawn(async move { inbox.push(1, 1, refused("second")).await }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!pushing.is_finished(), "the push waits for room"); + assert_eq!(inbox.take_all().len(), 1); + tokio::time::timeout(Duration::from_secs(5), pushing) + .await + .expect("the push finishes once room frees") + .expect("push task"); + assert_eq!(inbox.waiting(), 1); + } + + /// A stopped scheduler's inbox takes nothing, and a waiting push + /// returns. + #[tokio::test] + async fn a_dropped_handle_releases_a_waiting_push() { + let inboxes = CalvinInboxes::default(); + let handle = inboxes.register(5, 1); + let inbox = Arc::clone(handle.inbox()); + inbox.push(1, 0, refused("first")).await; + let pushing = { + let inbox = Arc::clone(&inbox); + tokio::spawn(async move { inbox.push(1, 1, refused("second")).await }) + }; + drop(handle); + tokio::time::timeout(Duration::from_secs(5), pushing) + .await + .expect("a closed inbox releases the push") + .expect("push task"); + assert!(inboxes.get(5).is_none()); + } + + /// A successor's registration replaces the inbox, and the predecessor's + /// drop leaves it in place. + #[test] + fn a_successor_keeps_its_inbox_when_the_predecessor_drops() { + let inboxes = CalvinInboxes::default(); + let first = inboxes.register(2, 4); + let second = inboxes.register(2, 4); + drop(first); + let current = inboxes.get(2).expect("the successor's inbox"); + assert!(Arc::ptr_eq(¤t, second.inbox())); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/metrics.rs b/nodedb/src/control/cluster/calvin/scheduler/metrics.rs index 6371d5451..b89b1e766 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/metrics.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/metrics.rs @@ -41,7 +41,7 @@ pub struct SchedulerMetrics { /// Times a staged txn parked in `AwaitingVerdict` passed its stall deadline /// with the durable global verdict still unknown. This is NOT an abort: the /// scheduler keeps waiting and holding locks (a unilateral abort while a - /// peer can already have flushed a commit will tear the transaction). A + /// peer can already have committed will tear the transaction). A /// non-zero, growing value flags a stuck sequencer / partitioned verdict /// path that needs operator attention, never a correctness action here. pub verdict_stall_count: AtomicU64, @@ -122,26 +122,28 @@ pub mod apply_halt_reason { pub const DISPATCH_REFUSED: usize = 1; pub const RESPONSE_DISCONNECTED: usize = 2; pub const RESOLVE_FAILED: usize = 3; - pub const FLUSH_FAILED: usize = 4; - pub const LOCAL_STAGE_FAILED: usize = 5; - pub const IDENTITY_BIND_FAILED: usize = 6; - pub const WAL_APPEND_FAILED: usize = 7; - pub const METADATA_GROUP_GONE: usize = 8; - pub const WRITE_VERSION_RECORD_FAILED: usize = 9; - pub const STAGE_VOTE_INVALID: usize = 10; + pub const LOCAL_STAGE_FAILED: usize = 4; + pub const IDENTITY_BIND_FAILED: usize = 5; + pub const METADATA_GROUP_GONE: usize = 6; + pub const STAGE_VOTE_INVALID: usize = 7; + pub const REDO_PROPOSE_FAILED: usize = 8; + pub const REDO_INSTALL_REFUSED: usize = 9; + pub const REDO_APPLY_FAILED: usize = 10; + pub const CORE_FAIL_STOPPED: usize = 11; pub const LABELS: &[&str] = &[ "draining", "dispatch_refused", "response_disconnected", "resolve_failed", - "flush_failed", "local_stage_failed", "identity_bind_failed", - "wal_append_failed", "metadata_group_gone", - "write_version_record_failed", "stage_vote_invalid", + "redo_propose_failed", + "redo_install_refused", + "redo_apply_failed", + "core_fail_stopped", ]; } diff --git a/nodedb/src/control/cluster/calvin/scheduler/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/mod.rs index 0f194a855..4e4fd83d6 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/mod.rs @@ -5,6 +5,7 @@ pub mod applied_ledger; pub mod caught_up; pub mod cut_floor; pub mod driver; +pub mod inbox; pub mod lock; pub mod metrics; mod metrics_flow; @@ -17,6 +18,7 @@ pub use driver::{ CalvinReadResultProposal, RaftSequencerProposer, ReadResultEvent, Scheduler, SchedulerConfig, SchedulerParams, SequencerProposer, propose_calvin_read_result, }; +pub use inbox::{CalvinApplyEvent, CalvinInboxes, CalvinVShardInbox, InboxHandle}; pub use lock::{AcquireOutcome, HotKeyTable, LockKey, LockManager, LockMode, TxnId}; // Existing call sites reference this module as `scheduler::lock_manager::…`; // keep that path stable via an alias while the module lives under `lock/`. diff --git a/nodedb/src/control/cluster/calvin/scheduler/recovery.rs b/nodedb/src/control/cluster/calvin/scheduler/recovery.rs index f5fdb076d..46e13c864 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/recovery.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/recovery.rs @@ -349,8 +349,6 @@ mod tests { epoch: 1, position: 1, vshard_id: vshard, - collections: Vec::new(), - sum_targets: Vec::new(), }), cross_shard_applied: None, row_sources: Vec::new(), diff --git a/nodedb/src/control/cluster/data_plane_error_wire.rs b/nodedb/src/control/cluster/data_plane_error_wire.rs index e4d6de18c..52b3c9395 100644 --- a/nodedb/src/control/cluster/data_plane_error_wire.rs +++ b/nodedb/src/control/cluster/data_plane_error_wire.rs @@ -38,6 +38,10 @@ impl From for DataPlaneErrorCode { } ErrorCode::RejectedPrevalidation { reason } => Self::RejectedPrevalidation { reason }, ErrorCode::RetryableRefusal { reason } => Self::RetryableRefusal { reason }, + ErrorCode::CoreFailStopped { core_id, detail } => Self::CoreFailStopped { + core_id: to_wire_count(core_id), + detail, + }, ErrorCode::SyncRejected { violation, applied_seq, @@ -186,6 +190,10 @@ impl From for ErrorCode { Self::RejectedPrevalidation { reason } } DataPlaneErrorCode::RetryableRefusal { reason } => Self::RetryableRefusal { reason }, + DataPlaneErrorCode::CoreFailStopped { core_id, detail } => Self::CoreFailStopped { + core_id: from_wire_count(core_id), + detail, + }, DataPlaneErrorCode::SyncRejected { violation, applied_seq, @@ -373,6 +381,16 @@ mod tests { } } + #[test] + fn a_core_fail_stop_roundtrips_verbatim() { + let original = ErrorCode::CoreFailStopped { + core_id: 3, + detail: "rollback failed".into(), + }; + let wire = DataPlaneErrorCode::from(original.clone()); + assert_eq!(ErrorCode::from(wire), original); + } + #[test] fn division_by_zero_survives_the_wire_hop() { let wire = DataPlaneErrorCode::from(ErrorCode::DivisionByZero); diff --git a/nodedb/src/control/cluster/sequencer_halt.rs b/nodedb/src/control/cluster/sequencer_halt.rs index ff4d9e466..f4c3ff903 100644 --- a/nodedb/src/control/cluster/sequencer_halt.rs +++ b/nodedb/src/control/cluster/sequencer_halt.rs @@ -122,9 +122,9 @@ mod tests { vshard_id, epoch: 9, position: 1, - reason: "flush_failed", - step: "flush", - error: "flush returned Error".to_string(), + reason: "redo_apply_failed", + step: "redo_apply", + error: "redo install did not become durable".to_string(), } } diff --git a/nodedb/src/control/cluster/start_raft/hooks.rs b/nodedb/src/control/cluster/start_raft/hooks.rs index e4bde7639..19ee735a9 100644 --- a/nodedb/src/control/cluster/start_raft/hooks.rs +++ b/nodedb/src/control/cluster/start_raft/hooks.rs @@ -102,6 +102,17 @@ pub(super) async fn build_hooks( .applied .install(vshard_id, state.fully_applied_epoch, state.applied_tail); } + // A chunk stream of an applied position installs nothing more: a copy + // of the position's redo installed already. + shared + .redo_chunks + .drop_applied_calvin_streams(|vshard_id, epoch, position| { + shared + .calvin + .applied + .get(vshard_id) + .is_some_and(|ledger| ledger.is_applied(epoch, position)) + }); // Per-group snapshot builder for the SEND path: on the leader, build the // real serialized engine state for a lagging follower's group vshards diff --git a/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs b/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs new file mode 100644 index 000000000..e80324678 --- /dev/null +++ b/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The Calvin part of a stamped `TransactionRedo` entry's apply. +//! +//! A committed Calvin slice installs only from its vShard's data-group log. +//! Every replica applies its entries in log order, so every replica decides +//! each copy of a slice's redo alike: +//! +//! 1. At entry start, in log order, the apply claims `(epoch, position)` in +//! the vShard's applied ledger. A claim refused because the position is +//! applied, or another copy holds it, concludes the entry applied and +//! installs nothing: a re-proposed copy never installs twice. +//! 2. The install runs like every committed redo's. +//! 3. When the install is durable, the ledger marks the position, the +//! position's open chunk streams drop, and the vShard's scheduler hears +//! `RedoApplied`. All of it happens before the entry concludes, so the +//! ledger holds the position before the group's applied index passes +//! the entry. +//! 4. A final refusal keeps the claim: every replica refuses the same entry, +//! and no later copy installs. An install that is not durable releases +//! the claim: a restart applies the entry again. +//! +//! A full inbox makes the apply wait for room. The entry holds its own +//! place in the group's lane meanwhile, as a parked write does. A vShard +//! with no running scheduler gets no event: the next scheduler seeds from +//! the ledger. + +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use crate::bridge::envelope::{ErrorCode, Payload, Response, Status}; +use crate::control::cluster::calvin::scheduler::{CalvinAppliedLedger, CalvinApplyEvent}; +use crate::control::server::dispatch_utils::{SubmitOutcome, refusal_is_final}; +use crate::control::state::SharedState; +use crate::control::wal_replication::types::CalvinRedoMeta; +use crate::types::{Lsn, RequestId}; +use crate::wal::RedoStreamId; + +/// A Calvin position this entry holds the claim on. +pub(super) struct CalvinClaim { + vshard_id: u32, + epoch: u64, + position: u32, + primary_write: bool, + returning: bool, + ledger: Arc, +} + +/// What a stamped redo entry's claim decided. +pub(super) enum ClaimOutcome { + /// The entry carries no Calvin slice. + NotCalvin, + /// The entry installs the position. + Claimed(CalvinClaim), + /// The position is applied, or another copy holds it: the entry + /// installs nothing. + Refused, + /// The entry names a Calvin slice but no position, so no replica can + /// install it. + Malformed(crate::Error), +} + +/// Claim the position of an inline Calvin redo: its record's stamp. +pub(super) fn claim_inline( + state: &SharedState, + meta: Option<&CalvinRedoMeta>, + stamp: Option<&crate::wal::CalvinStamp>, +) -> ClaimOutcome { + let Some(meta) = meta else { + return ClaimOutcome::NotCalvin; + }; + match stamp { + Some(stamp) => claim(state, meta, stamp.vshard_id, stamp.epoch, stamp.position), + None => ClaimOutcome::Malformed(crate::Error::Internal { + detail: "a Calvin redo entry carries no calvin_stamp, so it names no position".into(), + }), + } +} + +/// Claim the position of a chunked Calvin redo: its stream's. +pub(super) fn claim_chunked( + state: &SharedState, + meta: Option<&CalvinRedoMeta>, + stream: &RedoStreamId, +) -> ClaimOutcome { + let Some(meta) = meta else { + return ClaimOutcome::NotCalvin; + }; + match *stream { + RedoStreamId::Calvin { + vshard, + epoch, + position, + .. + } => claim(state, meta, vshard, epoch, position), + RedoStreamId::Session { .. } => ClaimOutcome::Malformed(crate::Error::Internal { + detail: format!("a Calvin redo entry names the session stream {stream:?}"), + }), + } +} + +fn claim( + state: &SharedState, + meta: &CalvinRedoMeta, + vshard_id: u32, + epoch: u64, + position: u32, +) -> ClaimOutcome { + let ledger = state.calvin.applied.get_or_create(vshard_id); + match ledger.claim(epoch, position) { + Ok(()) => ClaimOutcome::Claimed(CalvinClaim { + vshard_id, + epoch, + position, + primary_write: meta.primary_write, + returning: meta.returning, + ledger, + }), + Err(refusal) => { + tracing::debug!( + vshard_id, + epoch, + position, + ?refusal, + "calvin redo: a copy of an applied or claimed position installs nothing" + ); + ClaimOutcome::Refused + } + } +} + +impl CalvinClaim { + /// Conclude the claim from the install's outcome, and return what the + /// vShard's scheduler hears. + pub(super) fn settle( + &self, + state: &SharedState, + submitted: &crate::Result, + ) -> CalvinApplyEvent { + match submitted { + Ok(outcome) if outcome.response.status == Status::Ok => { + self.applied(state, outcome.response.clone()) + } + Ok(outcome) + if outcome + .response + .error_code + .as_deref() + .is_some_and(refusal_is_final) => + { + CalvinApplyEvent::RedoRefused { + error: format!("{:?}", outcome.response.error_code.as_deref()), + } + } + Ok(outcome) => self.not_applied(format!( + "the install answered {:?} with {:?}", + outcome.response.status, + outcome.response.error_code.as_deref() + )), + Err(error) => self.not_applied(error.to_string()), + } + } + + /// The entry's collection is gone on this replica: nothing installs, + /// and the position is done. + pub(super) fn superseded(&self, state: &SharedState) -> CalvinApplyEvent { + let reply = Response { + request_id: RequestId::new(0), + status: Status::Error, + attempt: 1, + partial: false, + payload: Payload::empty(), + watermark_lsn: Lsn::ZERO, + error_code: Some(Box::new(ErrorCode::NotFound)), + stage_vote: None, + read_version_lsn: Lsn::ZERO, + write_set: Vec::new(), + }; + self.applied(state, reply) + } + + /// The install did not run, or did not become durable. + pub(super) fn not_applied(&self, error: String) -> CalvinApplyEvent { + self.ledger.release_claim(self.epoch, self.position); + CalvinApplyEvent::RedoNotApplied { error } + } + + fn applied(&self, state: &SharedState, reply: Response) -> CalvinApplyEvent { + self.ledger.mark_applied(self.epoch, self.position); + state + .redo_chunks + .drop_calvin_position(self.vshard_id, self.epoch, self.position); + // The install recorded the slice's write versions. + state + .calvin + .counters + .write_versions_recorded + .fetch_add(1, Ordering::Relaxed); + CalvinApplyEvent::RedoApplied { + reply, + primary_write: self.primary_write, + returning: self.returning, + } + } + + /// Hand `event` to the vShard's scheduler, waiting for room in its + /// inbox. A vShard with no running scheduler takes nothing. + pub(super) async fn report(&self, state: &SharedState, event: CalvinApplyEvent) { + let Some(inbox) = state.calvin.inboxes.get(self.vshard_id) else { + return; + }; + inbox.push(self.epoch, self.position, event).await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::wal::WalManager; + use nodedb_physical::physical_plan::CalvinReplySpec; + + fn state() -> (Arc, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + (SharedState::new(dispatcher, wal).expect("state"), dir) + } + + fn meta() -> CalvinRedoMeta { + CalvinRedoMeta { + epoch_system_ms: 0, + reply: CalvinReplySpec::Count(Vec::new()), + primary_write: true, + user_write: true, + returning: false, + } + } + + fn stamp(position: u32) -> crate::wal::CalvinStamp { + crate::wal::CalvinStamp { + epoch: 4, + position, + vshard_id: 3, + } + } + + fn ok() -> crate::Result { + Ok(SubmitOutcome { + response: Response { + request_id: RequestId::new(1), + status: Status::Ok, + attempt: 1, + partial: false, + payload: Payload::empty(), + watermark_lsn: Lsn::ZERO, + error_code: None, + stage_vote: None, + read_version_lsn: Lsn::ZERO, + write_set: Vec::new(), + }, + }) + } + + /// A second stamped copy of a position finds the claim of the first, or + /// the position applied, and installs nothing. + #[test] + fn a_second_stamped_copy_installs_nothing() { + let (state, _dir) = state(); + let ClaimOutcome::Claimed(first) = claim_inline(&state, Some(&meta()), Some(&stamp(1))) + else { + panic!("the first copy claims the position"); + }; + assert!(matches!( + claim_inline(&state, Some(&meta()), Some(&stamp(1))), + ClaimOutcome::Refused + )); + let event = first.settle(&state, &ok()); + assert!(matches!(event, CalvinApplyEvent::RedoApplied { .. })); + assert!(matches!( + claim_inline(&state, Some(&meta()), Some(&stamp(1))), + ClaimOutcome::Refused + )); + assert!(matches!( + claim_chunked( + &state, + Some(&meta()), + &RedoStreamId::Calvin { + vshard: 3, + epoch: 4, + position: 1, + attempt: 2, + } + ), + ClaimOutcome::Refused + )); + } + + /// The ledger holds a durable install's position by the time its + /// settle returns, before the entry concludes and the group's applied + /// index passes it. An install that is not durable releases the claim. + #[test] + fn the_ledger_marks_before_the_applied_index_passes() { + let (state, _dir) = state(); + let ledger = state.calvin.applied.get_or_create(3); + let ClaimOutcome::Claimed(claim) = claim_inline(&state, Some(&meta()), Some(&stamp(2))) + else { + panic!("claims"); + }; + assert!( + !ledger.is_applied(4, 2), + "a claim is not an applied position" + ); + claim.settle(&state, &ok()); + assert!(ledger.is_applied(4, 2)); + + let ClaimOutcome::Claimed(failing) = claim_inline(&state, Some(&meta()), Some(&stamp(5))) + else { + panic!("claims"); + }; + let event = failing.settle( + &state, + &Err(crate::Error::Internal { + detail: "fsync failed".into(), + }), + ); + assert!(matches!(event, CalvinApplyEvent::RedoNotApplied { .. })); + assert!(!ledger.is_applied(4, 5)); + assert!( + matches!( + claim_inline(&state, Some(&meta()), Some(&stamp(5))), + ClaimOutcome::Claimed(_) + ), + "a released claim lets the next copy install" + ); + } + + /// An entry with no Calvin meta claims nothing, and a Calvin meta with + /// no position is malformed. + #[test] + fn only_a_stamped_calvin_entry_claims() { + let (state, _dir) = state(); + assert!(matches!( + claim_inline(&state, None, Some(&stamp(1))), + ClaimOutcome::NotCalvin + )); + assert!(matches!( + claim_inline(&state, Some(&meta()), None), + ClaimOutcome::Malformed(_) + )); + } + + /// A vShard with no running scheduler hears nothing, and its ledger + /// still holds the position. + #[tokio::test] + async fn no_scheduler_takes_no_event() { + let (state, _dir) = state(); + let ClaimOutcome::Claimed(claim) = claim_inline(&state, Some(&meta()), Some(&stamp(3))) + else { + panic!("claims"); + }; + let event = claim.settle(&state, &ok()); + claim.report(&state, event).await; + assert!(state.calvin.applied.get_or_create(3).is_applied(4, 3)); + + let handle = state.calvin.inboxes.register(3, 4); + let ClaimOutcome::Claimed(next) = claim_inline(&state, Some(&meta()), Some(&stamp(4))) + else { + panic!("claims"); + }; + let event = next.settle(&state, &ok()); + next.report(&state, event).await; + assert_eq!(handle.inbox().take_all().len(), 1); + } +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/lane.rs b/nodedb/src/control/distributed_applier/apply_loop/lane.rs index e45dbe55e..956f0c28d 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/lane.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/lane.rs @@ -18,6 +18,7 @@ use crate::control::distributed_applier::applied_index::AppliedPrefix; use crate::control::distributed_applier::propose_tracker::{ApplyingEntry, ProposeTracker}; use crate::control::server::shared::write_admission::plan_writes_user_data; use crate::control::state::tenant_marks::{MarkSite, TenantMarks}; +use crate::control::wal_replication::types::CalvinRedoMeta; use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite, decode_replicated_entry}; use super::proposal_gate::PrefixStep; @@ -53,8 +54,9 @@ impl QueuedEntry { /// `(tenant_id, write_hlc, restore_id)` of an entry that writes a /// tenant's data. A cut barrier, a Calvin read result and a surrogate - /// bind write no data, and an entry with no proposer stamp has no commit - /// HLC to record. + /// bind write no data, a Calvin slice that raises no write mark writes + /// only derived rows or schema, and an entry with no proposer stamp has + /// no commit HLC to record. pub fn write_stamp(&self) -> Option<(u64, u64, u64)> { let decoded = self.decoded.as_ref()?; if decoded.write_hlc == 0 @@ -65,6 +67,13 @@ impl QueuedEntry { | ReplicatedWrite::SurrogateBind { .. } | ReplicatedWrite::RedoChunk { .. } | ReplicatedWrite::RedoAbandon { .. } + | ReplicatedWrite::TransactionRedo { + calvin: Some(CalvinRedoMeta { + user_write: false, + .. + }), + .. + } ) { return None; @@ -79,11 +88,13 @@ impl QueuedEntry { let Some(decoded) = self.decoded.as_ref() else { return false; }; - match decoded.write { + match &decoded.write { + ReplicatedWrite::TransactionRedo { calvin, .. } => { + calvin.as_ref().is_none_or(|meta| meta.user_write) + } ReplicatedWrite::ArrayOp { .. } | ReplicatedWrite::ArrayCellPut { .. } - | ReplicatedWrite::ArrayCellDelete { .. } - | ReplicatedWrite::TransactionRedo { .. } => true, + | ReplicatedWrite::ArrayCellDelete { .. } => true, ReplicatedWrite::ArraySchema { .. } | ReplicatedWrite::CutBarrier { .. } | ReplicatedWrite::CalvinReadResult { .. } @@ -566,4 +577,55 @@ mod tests { lane.settle(&tracker, &TenantMarks::default()); assert!(matches!(barrier.try_recv(), Ok(Ok(_)))); } + + /// The stamped redo entry of a Calvin slice that raises the write mark + /// when `user_write` holds, proposed with commit HLC 77. + fn calvin_redo_entry(user_write: bool) -> QueuedEntry { + let payload = crate::control::wal_replication::transaction_redo::TransactionRedoPayload { + redo: crate::wal::RedoRecord { + version: 1, + ops: Vec::new(), + calvin_stamp: None, + cross_shard_applied: None, + row_sources: Vec::new(), + publishes: Vec::new(), + row_changes: Vec::new(), + }, + collections: vec!["orders".into()], + sum_targets: Vec::new(), + identities: Vec::new(), + event_source: crate::event::EventSource::User, + origin: nodedb_physical::physical_plan::RedoOrigin::Commit, + calvin: Some(CalvinRedoMeta { + epoch_system_ms: 0, + reply: nodedb_physical::physical_plan::CalvinReplySpec::Count(Vec::new()), + primary_write: true, + user_write, + returning: false, + }), + }; + let mut entry = crate::control::wal_replication::encode::transaction_redo_entry( + crate::types::TenantId::new(4), + crate::types::DatabaseId::DEFAULT, + crate::types::VShardId::new(2), + &payload, + ); + entry.write_hlc = 77; + QueuedEntry::new(LogEntry { + term: 1, + index: 9, + data: entry.to_bytes(), + }) + } + + /// A Calvin slice that writes only derived rows or schema raises no write + /// mark, even as the slice that deposits the reply. A slice that changes + /// a tenant row raises its commit HLC. + #[test] + fn only_a_calvin_slice_that_changes_a_row_carries_a_write_stamp() { + assert_eq!(calvin_redo_entry(true).write_stamp(), Some((4, 77, 0))); + assert_eq!(calvin_redo_entry(false).write_stamp(), None); + assert!(calvin_redo_entry(true).plan_writes_user_data()); + assert!(!calvin_redo_entry(false).plan_writes_user_data()); + } } diff --git a/nodedb/src/control/distributed_applier/apply_loop/mod.rs b/nodedb/src/control/distributed_applier/apply_loop/mod.rs index 0c6e74a43..a10a41bca 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/mod.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/mod.rs @@ -23,6 +23,8 @@ //! collects. //! - [`calvin_read_result`]: forwards a committed `CalvinReadResult` entry to //! the local Calvin scheduler. +//! - [`calvin_redo`]: claims a stamped Calvin redo's position in the applied +//! ledger and reports its install to the vShard's scheduler. //! - [`write_dispatch`]: the generic decode + write-funnel enqueue path. //! - [`transaction_redo`]: a committed transaction's redo, stamped with its //! Raft entry and applied through the WAL replay arms. @@ -47,6 +49,7 @@ mod array_cell_route; mod bookkeeping; mod calvin_read_result; +mod calvin_redo; mod collection_route; mod context; mod driver; diff --git a/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs b/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs index 44cdfe355..e9064afd4 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs @@ -26,10 +26,14 @@ use crate::types::{DatabaseId, TenantId, VShardId}; use crate::wal::RedoStreamId; use crate::wal::manager::RedoChunkPlacement; +use super::calvin_redo::{ClaimOutcome, claim_chunked}; use super::context::{ApplyContext, FinishedApply}; use super::proposal_gate::{EntryOutcome, ledger_outcome}; use super::start::Prepared; -use super::transaction_redo::{conclude_held, enqueue_payload}; +use super::transaction_redo::{ + RedoApply, conclude_held, conclude_held_claimed, conclude_installed_copy, enqueue_payload, + report_then_conclude, +}; /// Prepare a committed `RedoChunk` or `RedoAbandon` entry of term `term`. pub(super) fn prepare_stream_entry<'a>( @@ -202,7 +206,16 @@ pub(super) fn prepare_chunked_final<'a>( chunked: ChunkedRedo, incarnations: Vec, ) -> Prepared<'a> { + let claim = claim_chunked(ctx.state, chunked.calvin(), &chunked.stream); let taken = ctx.state.redo_chunks.take_for_final(&chunked.stream); + let calvin = match claim { + ClaimOutcome::NotCalvin => None, + ClaimOutcome::Claimed(claim) => Some(claim), + // Another copy of the position installed, or installs now: this + // copy's stream drops. + ClaimOutcome::Refused => return conclude_installed_copy(ctx, pos, taken), + ClaimOutcome::Malformed(error) => return conclude_held(ctx, pos, error, taken), + }; // Every chunk is durable and the final entry committed; nothing of the // final entry applied yet. crate::fail_point!("redo_chunk::before_final_install"); @@ -216,9 +229,14 @@ pub(super) fn prepare_chunked_final<'a>( Ok(bytes) => bytes, Err(refusal) if ctx.state.redo_chunks.owes_snapshot(pos.group_id) => { let error = stream_lost_here(pos, chunked.stream, LostEntry::Final, &refusal); - return conclude_held(ctx, pos, error, taken); + return match calvin { + Some(claim) => conclude_held_claimed(ctx, pos, error, taken, claim), + None => conclude_held(ctx, pos, error, taken), + }; } Err(refusal) => { + // Every replica refuses the final entry alike. The claim stays, + // and the scheduler hears the refusal once the entry starts. let result = Err(crate::Error::from(refusal)); let outcome = EntryOutcome::Applied { durable: true, @@ -226,7 +244,18 @@ pub(super) fn prepare_chunked_final<'a>( }; ctx.tracker .complete(pos.group_id, pos.log_index, pos.applied_key, result); - return Prepared::Concluded(outcome); + return match calvin { + Some(claim) => { + let event = + crate::control::cluster::calvin::scheduler::CalvinApplyEvent::RedoRefused { + error: "the final entry's chunk stream does not hold the chunks it \ + names" + .into(), + }; + report_then_conclude(ctx, pos, claim, event, outcome) + } + None => Prepared::Concluded(outcome), + }; } }; match RedoContent::from_bytes(&bytes) { @@ -234,11 +263,17 @@ pub(super) fn prepare_chunked_final<'a>( ctx, pos, target, - chunked.payload(content), - incarnations, - taken, + RedoApply { + payload: chunked.payload(content), + incarnations, + stream: taken, + calvin, + }, )), - Err(error) => conclude_held(ctx, pos, error, taken), + Err(error) => match calvin { + Some(claim) => conclude_held_claimed(ctx, pos, error, taken, claim), + None => conclude_held(ctx, pos, error, taken), + }, } } @@ -300,6 +335,7 @@ mod tests { collections: Vec::new(), event_source: ReplicatedEventSource::User, origin: RedoOrigin::Commit, + calvin: None, }; match decode_transaction_redo(&write).expect("decode") { DecodedTransactionRedo::Chunked(chunked) => chunked, diff --git a/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs b/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs index 54b213d74..094fda2fd 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs @@ -12,6 +12,10 @@ //! A chunked body assembles from the `RedoChunk` entries of its stream //! (see [`super::redo_chunk`]) and then applies like an inline one. //! +//! A committed Calvin slice's entry claims its position before it applies +//! and reports its install to the vShard's scheduler (see +//! [`super::calvin_redo`]). +//! //! A refusal the Data Plane proves applied nothing (a constraint verdict) is //! final: every replica reaches it at the same log position against the same //! state, and the funnel cancels the record in the WAL before it returns. The @@ -31,6 +35,7 @@ use crate::control::wal_replication::transaction_redo::{ use crate::control::wal_replication::{CollectionIncarnation, ReplicatedEntry}; use crate::types::{DatabaseId, TenantId, VShardId}; +use super::calvin_redo::{CalvinClaim, ClaimOutcome, claim_inline}; use super::context::{ApplyContext, EnqueueFuture, FinishedApply, Started, StartedEntry}; use super::helpers::committed_response_result; use super::proposal_gate::{EntryOutcome, ledger_outcome}; @@ -60,20 +65,68 @@ pub(super) fn prepare_transaction_redo_entry<'a>( vshard_id: VShardId::new(entry.vshard_id), }; match decoded { - DecodedTransactionRedo::Inline(payload) => Prepared::Enqueue(enqueue_payload( - ctx, - pos, - target, - *payload, - incarnations, - None, - )), + DecodedTransactionRedo::Inline(payload) => { + let claim = claim_inline( + ctx.state, + payload.calvin.as_ref(), + payload.redo.calvin_stamp.as_ref(), + ); + let calvin = match claim { + ClaimOutcome::NotCalvin => None, + ClaimOutcome::Claimed(claim) => Some(claim), + ClaimOutcome::Refused => return conclude_installed_copy(ctx, pos, None), + ClaimOutcome::Malformed(error) => return conclude_held(ctx, pos, error, None), + }; + Prepared::Enqueue(enqueue_payload( + ctx, + pos, + target, + RedoApply { + payload: *payload, + incarnations, + stream: None, + calvin, + }, + )) + } DecodedTransactionRedo::Chunked(chunked) => { super::redo_chunk::prepare_chunked_final(ctx, pos, target, chunked, incarnations) } } } +/// Conclude a copy of a Calvin position that another copy installed, or +/// installs now. It applies nothing, and its waiter learns the write +/// committed. `stream` is the chunk stream the copy took: it drops. +pub(super) fn conclude_installed_copy<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + stream: Option, +) -> Prepared<'a> { + ctx.tracker.complete( + pos.group_id, + pos.log_index, + pos.applied_key, + Ok(AppliedWrite::unversioned(Vec::new())), + ); + drop(stream); + Prepared::Concluded(EntryOutcome::Applied { + durable: true, + result: None, + }) +} + +/// One committed redo on its way to its core. +pub(super) struct RedoApply { + pub payload: TransactionRedoPayload, + /// The entry's collection incarnations, moved out of the decoded entry. + pub incarnations: Vec, + /// The chunk stream the payload assembled from. + pub stream: Option, + /// The Calvin position the entry claimed. + pub calvin: Option, +} + /// Conclude an entry this replica cannot apply as the other replicas do: /// its own bytes are malformed, or its stream here is not the one the log /// built. It holds the floor rather than skipping a committed transaction. @@ -84,6 +137,51 @@ pub(super) fn conclude_held<'a>( error: crate::Error, stream: Option, ) -> Prepared<'a> { + Prepared::Concluded(held_outcome(ctx, pos, error, stream)) +} + +/// [`conclude_held`] for an entry that claimed a Calvin position: the claim +/// releases and the vShard's scheduler hears the install did not apply. +pub(super) fn conclude_held_claimed<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + error: crate::Error, + stream: Option, + claim: CalvinClaim, +) -> Prepared<'a> { + let event = claim.not_applied(error.to_string()); + let outcome = held_outcome(ctx, pos, error, stream); + report_then_conclude(ctx, pos, claim, event, outcome) +} + +/// Report `event` to the claim's scheduler, then conclude the entry with +/// `outcome`. The report can wait for room in the scheduler's inbox, so the +/// entry runs as its own apply. +pub(super) fn report_then_conclude<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + claim: CalvinClaim, + event: crate::control::cluster::calvin::scheduler::CalvinApplyEvent, + outcome: EntryOutcome, +) -> Prepared<'a> { + Prepared::Exclusive(Box::pin(async move { + claim.report(ctx.state, event).await; + FinishedApply { + group_id: pos.group_id, + log_index: pos.log_index, + outcome, + } + })) +} + +/// Resolve the waiter of an entry this replica holds, and settle its +/// stream. +fn held_outcome( + ctx: ApplyContext<'_>, + pos: AppliedPosition, + error: crate::Error, + stream: Option, +) -> EntryOutcome { ctx.tracker .complete(pos.group_id, pos.log_index, pos.applied_key, Err(error)); let outcome = EntryOutcome::Applied { @@ -91,7 +189,7 @@ pub(super) fn conclude_held<'a>( result: None, }; settle_stream(ctx, stream, &outcome); - Prepared::Concluded(outcome) + outcome } /// Release a taken stream once its final entry concluded. A durable outcome @@ -106,19 +204,29 @@ fn settle_stream(ctx: ApplyContext<'_>, stream: Option, outcome: &En } } -/// Route `payload` to its collections, append it, and enqueue it on its -/// core. `stream` is the chunked stream the payload assembled from. +/// Route the redo to its collections, append it, and enqueue it on its +/// core. pub(super) fn enqueue_payload<'a>( ctx: ApplyContext<'a>, pos: AppliedPosition, target: RedoTarget, - payload: TransactionRedoPayload, - incarnations: Vec, - stream: Option, + apply: RedoApply, ) -> EnqueueFuture<'a> { let ApplyContext { state, tracker, .. } = ctx; + let RedoApply { + payload, + incarnations, + stream, + calvin, + } = apply; Box::pin(async move { let collection = payload.collections.first().cloned(); + // A Calvin slice with no primary write writes only derived rows: it + // raises no tenant write mark. + let user_write = payload + .calvin + .as_ref() + .is_none_or(|meta| meta.primary_write); // A redo for a collection incarnation this node no longer holds has // nothing to mutate. The gates stay held until the redo is enqueued. let routed = super::collection_route::route( @@ -131,6 +239,10 @@ pub(super) fn enqueue_payload<'a>( let _gates = match routed { Ok(super::collection_route::CollectionRoute::Apply(gates)) => gates, Ok(super::collection_route::CollectionRoute::Superseded) => { + if let Some(claim) = &calvin { + let event = claim.superseded(state); + claim.report(state, event).await; + } tracker.complete( pos.group_id, pos.log_index, @@ -147,6 +259,10 @@ pub(super) fn enqueue_payload<'a>( return StartedEntry::concluded(outcome); } Err(error) => { + if let Some(claim) = &calvin { + let event = claim.not_applied(error.to_string()); + claim.report(state, event).await; + } tracker.complete(pos.group_id, pos.log_index, pos.applied_key, Err(error)); let outcome = EntryOutcome::Applied { durable: false, @@ -177,6 +293,12 @@ pub(super) fn enqueue_payload<'a>( if let Ok(outcome) = &submitted { record_cross_shard_key(state, &payload, outcome); } + // The ledger holds the position before the entry concludes, + // so before the group's applied index passes it. + if let Some(claim) = &calvin { + let event = claim.settle(state, &submitted); + claim.report(state, event).await; + } let outcome = conclude_transaction_redo(tracker, pos, submitted); settle_stream(ctx, stream, &outcome); FinishedApply { @@ -186,6 +308,10 @@ pub(super) fn enqueue_payload<'a>( } })), Err(error) => { + if let Some(claim) = &calvin { + let event = claim.not_applied(error.to_string()); + claim.report(state, event).await; + } let outcome = conclude_transaction_redo(tracker, pos, Err(error)); settle_stream(ctx, stream, &outcome); Started::Concluded(outcome) @@ -195,7 +321,7 @@ pub(super) fn enqueue_payload<'a>( StartedEntry { started, collection, - user_write: true, + user_write, } }) } diff --git a/nodedb/src/control/fail_gate.rs b/nodedb/src/control/fail_gate.rs index d8ed2319c..6bd860104 100644 --- a/nodedb/src/control/fail_gate.rs +++ b/nodedb/src/control/fail_gate.rs @@ -73,20 +73,33 @@ pub(crate) async fn after_calvin_stamp(tx_class: &nodedb_cluster::calvin::types: } } -/// The Calvin scheduler's flush gate: after a committed transaction's redo -/// record is appended and before its flush reaches a core. Named per -/// collection, `calvin::before_flush::`. +/// The Calvin scheduler's redo gate: after a committed slice's redo is +/// resolved and before the data-group leader proposes it. Named per +/// collection, `calvin::before_redo_propose::`. /// /// The scheduler cannot park on a timer, so the gate answers without waiting: -/// `true` while the file armed for one of the flush's collections is absent. -/// The scheduler then keeps the flush in its re-send queue and asks again on -/// its next pass. -pub(crate) fn holds_flush(plan: &PhysicalPlan) -> bool { - plan.named_collections().iter().any(|collection| { - matches!( - lookup(&format!("calvin::before_flush::{collection}")), - Some(FailAction::WaitForFile(path)) if !path.exists() - ) +/// `true` while the file armed for one of `collections` is absent. The +/// scheduler then keeps the slice unproposed and asks again on its next stall +/// tick. On its first hold the gate creates `.parked`, so a test can +/// wait until the slice is held. +pub(crate) fn holds_redo_propose(collections: &[String]) -> bool { + collections.iter().any(|collection| { + let name = format!("calvin::before_redo_propose::{collection}"); + let Some(FailAction::WaitForFile(path)) = lookup(&name) else { + return false; + }; + if path.exists() { + return false; + } + let mut parked = path.into_os_string(); + parked.push(".parked"); + let parked = std::path::PathBuf::from(parked); + if !parked.exists() + && let Err(e) = std::fs::write(&parked, &name) + { + tracing::warn!(gate = %name, error = %e, "fail gate parked marker not created"); + } + true }) } diff --git a/nodedb/src/control/gateway/error_map/class_parity/code_samples.rs b/nodedb/src/control/gateway/error_map/class_parity/code_samples.rs index 5b67f3f64..aa3a526bb 100644 --- a/nodedb/src/control/gateway/error_map/class_parity/code_samples.rs +++ b/nodedb/src/control/gateway/error_map/class_parity/code_samples.rs @@ -8,7 +8,7 @@ use nodedb_types::sync::wire::SyncProvenance; use crate::bridge::envelope::{CounterFault, ErrorCode, SyncHold}; /// The number of `ErrorCode` variants [`variant_index`] numbers. -pub(super) const VARIANT_COUNT: usize = 52; +pub(super) const VARIANT_COUNT: usize = 53; /// A dense index per variant. Exhaustive, so a new variant fails to compile /// here until it gets an index, and [`every_variant_has_a_sample`] then fails @@ -67,6 +67,7 @@ pub(super) fn variant_index(code: &ErrorCode) -> usize { ErrorCode::DatetimeFieldOverflow { .. } => 49, ErrorCode::UndefinedObject { .. } => 50, ErrorCode::ObjectNotInPrerequisiteState { .. } => 51, + ErrorCode::CoreFailStopped { .. } => 52, } } @@ -88,6 +89,10 @@ pub(super) fn samples() -> Vec { ErrorCode::DeadlineExceeded, ErrorCode::RejectedPrevalidation { reason: text() }, ErrorCode::RetryableRefusal { reason: text() }, + ErrorCode::CoreFailStopped { + core_id: 0, + detail: text(), + }, ErrorCode::SyncRejected { violation: ViolationType::PermissionDenied, applied_seq: 1, diff --git a/nodedb/src/control/planner/rls_injection/meta.rs b/nodedb/src/control/planner/rls_injection/meta.rs index 1506be6b1..2d7f90e7b 100644 --- a/nodedb/src/control/planner/rls_injection/meta.rs +++ b/nodedb/src/control/planner/rls_injection/meta.rs @@ -57,7 +57,6 @@ pub(super) fn inject_meta(ctx: &RlsCtx<'_>, op: &mut MetaOp) -> crate::Result<() MetaOp::TransactionBatch { plans, .. } | MetaOp::CalvinExecuteStatic { plans, .. } | MetaOp::CalvinExecuteActive { plans, .. } - | MetaOp::RecordCalvinWriteVersions { plans, .. } | MetaOp::ResolveTxn { plans, .. } => { for plan in plans.iter_mut() { walk(ctx, plan)?; diff --git a/nodedb/src/control/planner/rls_injection/permission_tree/meta.rs b/nodedb/src/control/planner/rls_injection/permission_tree/meta.rs index 9d52694a0..6136fb15c 100644 --- a/nodedb/src/control/planner/rls_injection/permission_tree/meta.rs +++ b/nodedb/src/control/planner/rls_injection/permission_tree/meta.rs @@ -57,7 +57,6 @@ pub(super) fn apply_meta(ctx: &PermCtx<'_>, op: &mut MetaOp) -> crate::Result<() MetaOp::TransactionBatch { plans, .. } | MetaOp::CalvinExecuteStatic { plans, .. } | MetaOp::CalvinExecuteActive { plans, .. } - | MetaOp::RecordCalvinWriteVersions { plans, .. } | MetaOp::ResolveTxn { plans, .. } => { for plan in plans.iter_mut() { walk(ctx, plan)?; diff --git a/nodedb/src/control/security/identity/plan_permission.rs b/nodedb/src/control/security/identity/plan_permission.rs index 9e826ab7f..ae2467d47 100644 --- a/nodedb/src/control/security/identity/plan_permission.rs +++ b/nodedb/src/control/security/identity/plan_permission.rs @@ -394,7 +394,6 @@ pub fn required_permission(plan: &crate::bridge::envelope::PhysicalPlan) -> Perm MetaOp::CalvinExecuteStatic { .. } | MetaOp::CalvinExecutePassive { .. } | MetaOp::CalvinExecuteActive { .. } - | MetaOp::RecordCalvinWriteVersions { .. } | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. }, ) => Permission::Write, diff --git a/nodedb/src/control/server/dispatch_utils/change_events/mod.rs b/nodedb/src/control/server/dispatch_utils/change_events/mod.rs index 3d30d142e..f968a6adf 100644 --- a/nodedb/src/control/server/dispatch_utils/change_events/mod.rs +++ b/nodedb/src/control/server/dispatch_utils/change_events/mod.rs @@ -8,7 +8,4 @@ mod extract; mod publish; mod redo; -pub(crate) use publish::{ - CalvinApply, PendingChanges, WriteChangeSet, extract_write_change_set, - publish_calvin_change_sets, publish_settled_changes, redo_change_set, -}; +pub(crate) use publish::{PendingChanges, extract_write_change_set, publish_settled_changes}; diff --git a/nodedb/src/control/server/dispatch_utils/change_events/publish.rs b/nodedb/src/control/server/dispatch_utils/change_events/publish.rs index 4a8c4db9e..2b73cf57b 100644 --- a/nodedb/src/control/server/dispatch_utils/change_events/publish.rs +++ b/nodedb/src/control/server/dispatch_utils/change_events/publish.rs @@ -7,9 +7,8 @@ use std::sync::Arc; use crate::bridge::envelope::{PhysicalPlan, Response}; -use crate::control::change_stream::{ChangeEvent, ChangePartition, ChangeRun, PositionedChange}; +use crate::control::change_stream::{ChangeEvent, ChangePartition}; use crate::control::state::SharedState; -use crate::event::cdc::CdcOffset; use crate::types::{DatabaseId, TenantId}; use super::extract::{WriteChangeMeta, extract_write_metadata}; @@ -111,15 +110,6 @@ pub(crate) fn extract_write_change_set(plan: &PhysicalPlan, tenant_id: TenantId) } } -/// The change events of a committed transaction's redo record `redo`: one per -/// row it installs. A Calvin commit publishes these, as the data-group apply -/// of a `TransactionRedo` entry does. -pub(crate) fn redo_change_set(redo: &[u8]) -> WriteChangeSet { - WriteChangeSet { - metas: super::redo::redo_change_meta(redo), - } -} - fn change_events( shared: &SharedState, tenant_id: TenantId, @@ -216,62 +206,3 @@ pub(crate) fn publish_settled_changes( .change_stream .forward(shared, ChangePartition::Group(group_id)); } - -/// Publish the changes of the Calvin transaction at `(sequencer_epoch, -/// position)` that vShard `vshard` applied, and forward them when this node -/// leads the vShard's data group. -/// -/// Every replica's scheduler applies the vShard's transactions in sequencer -/// order, so every replica publishes them at the same positions. -pub(crate) fn publish_calvin_change_sets( - shared: &Arc, - calvin: CalvinApply, - change_sets: Vec, - lsn: nodedb_types::Lsn, -) { - let CalvinApply { - tenant_id, - database_id, - vshard, - sequencer_epoch, - position, - commit_hlc, - } = calvin; - let base = u64::from(position); - let changes: Vec = change_sets - .into_iter() - .flat_map(|change_set| { - change_events(shared, tenant_id, database_id, change_set, lsn, commit_hlc) - }) - .enumerate() - .map(|(ordinal, event)| PositionedChange { - position: CdcOffset::data_event_in(0, sequencer_epoch, base, ordinal as u64 + 1), - database_id, - event, - }) - .collect(); - if changes.is_empty() { - return; - } - shared.change_stream.publish_run(ChangeRun { - partition: ChangePartition::Calvin(vshard), - after: None, - through: CdcOffset::data_event_in(0, sequencer_epoch, base, u64::MAX).correction(), - changes, - }); - shared - .change_stream - .forward(shared, ChangePartition::Calvin(vshard)); -} - -/// The Calvin transaction a published change set belongs to. -#[derive(Debug, Clone, Copy)] -pub(crate) struct CalvinApply { - pub tenant_id: TenantId, - pub database_id: DatabaseId, - pub vshard: u32, - pub sequencer_epoch: u64, - pub position: u32, - /// The transaction's commit HLC, which every replica derives alike. - pub commit_hlc: u64, -} diff --git a/nodedb/src/control/server/dispatch_utils/minted/records.rs b/nodedb/src/control/server/dispatch_utils/minted/records.rs index d0fd26aa8..67d14b737 100644 --- a/nodedb/src/control/server/dispatch_utils/minted/records.rs +++ b/nodedb/src/control/server/dispatch_utils/minted/records.rs @@ -141,11 +141,6 @@ impl MintedRecords { self.sent.store(true, Ordering::Release); } - /// The highest LSN appended under this window, `None` before any append. - pub(crate) fn last_lsn(&self) -> Option { - self.recorded().iter().map(|record| record.lsn).max() - } - /// Every appended LSN, in append order. #[cfg(test)] pub(crate) fn lsns(&self) -> Vec { diff --git a/nodedb/src/control/server/dispatch_utils/mod.rs b/nodedb/src/control/server/dispatch_utils/mod.rs index 6582a3fe4..ab3c59b3a 100644 --- a/nodedb/src/control/server/dispatch_utils/mod.rs +++ b/nodedb/src/control/server/dispatch_utils/mod.rs @@ -13,10 +13,7 @@ mod types; mod unlogged_dispatch; mod write_abort; -pub(crate) use change_events::{ - CalvinApply, WriteChangeSet, publish_calvin_change_sets, publish_settled_changes, - redo_change_set, -}; +pub(crate) use change_events::publish_settled_changes; pub use dispatch::{dispatch_authorized_autocommit_write, dispatch_authorized_to_data_plane}; pub(crate) use dispatch::{ dispatch_autocommit_write, dispatch_replayed_write_to_data_plane, dispatch_to_data_plane, diff --git a/nodedb/src/control/server/dispatch_utils/write_abort.rs b/nodedb/src/control/server/dispatch_utils/write_abort.rs index 0ed1cdba0..088202080 100644 --- a/nodedb/src/control/server/dispatch_utils/write_abort.rs +++ b/nodedb/src/control/server/dispatch_utils/write_abort.rs @@ -36,9 +36,8 @@ use crate::bridge::envelope::ErrorCode; /// a redelivery here can too. That covers admission and capacity verdicts, /// a task that expired before it started, concurrency retries, the staging /// byte budget, a sync hold, which depends on the core's own stream mark, -/// and `RetryableRefusal`, -/// which a committed-redo apply answers with after it rolled a failed -/// install back. +/// `RetryableRefusal`, which a committed-redo apply answers with after it +/// rolled a failed install back, and a fail-stopped core's refusal. pub(crate) fn refusal_is_final(code: &ErrorCode) -> bool { write_definitely_not_applied(code) && !is_transient_verdict(code) } @@ -48,6 +47,7 @@ pub(crate) fn refusal_is_final(code: &ErrorCode) -> bool { fn is_transient_verdict(code: &ErrorCode) -> bool { match code { ErrorCode::RetryableRefusal { .. } + | ErrorCode::CoreFailStopped { .. } | ErrorCode::SyncNotApplied { .. } | ErrorCode::RateExceeded { .. } | ErrorCode::CollectionDraining { .. } @@ -149,6 +149,8 @@ pub(crate) fn write_definitely_not_applied(code: &ErrorCode) -> bool { // Documented as applying nothing: the identical frame is expected to be // re-sent, and the retry carries its own record. | ErrorCode::RetryableRefusal { .. } + // A fail-stopped core refuses every request before it runs. + | ErrorCode::CoreFailStopped { .. } // Concurrency verdicts that abort the whole attempt before install. | ErrorCode::ConflictRetry | ErrorCode::OllpRetryRequired diff --git a/nodedb/src/control/server/exchange/all_cores/dispatch.rs b/nodedb/src/control/server/exchange/all_cores/dispatch.rs index 5a720abef..a7db1a2a5 100644 --- a/nodedb/src/control/server/exchange/all_cores/dispatch.rs +++ b/nodedb/src/control/server/exchange/all_cores/dispatch.rs @@ -195,7 +195,6 @@ pub(crate) async fn execute_plan_all_local_cores( | MetaOp::RebuildIndex { .. } | MetaOp::PutSynonymGroup { .. } | MetaOp::DeleteSynonymGroup { .. } - | MetaOp::RecordCalvinWriteVersions { .. } | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. } => { generic_gather(state, tenant_id, database_id, plan, trace_id, txn_id).await diff --git a/nodedb/src/control/server/shared/authorization/requirements/collect.rs b/nodedb/src/control/server/shared/authorization/requirements/collect.rs index b2a72725a..5ca6a3bf4 100644 --- a/nodedb/src/control/server/shared/authorization/requirements/collect.rs +++ b/nodedb/src/control/server/shared/authorization/requirements/collect.rs @@ -108,7 +108,6 @@ fn collect_requirements(plan: &PhysicalPlan, out: &mut Vec add_collection_requirement(name, required_permission(plan), out), PhysicalPlan::Meta(MetaOp::TransactionBatch { plans, .. }) | PhysicalPlan::Meta(MetaOp::ResolveTxn { plans, .. }) - | PhysicalPlan::Meta(MetaOp::RecordCalvinWriteVersions { plans, .. }) | PhysicalPlan::Meta(MetaOp::CalvinExecuteStatic { plans, .. }) | PhysicalPlan::Meta(MetaOp::CalvinExecuteActive { plans, .. }) => { for nested in plans { @@ -252,7 +251,6 @@ fn requires_tenant_fallback(plan: &PhysicalPlan) -> bool { PhysicalPlan::Meta( MetaOp::TransactionBatch { plans, .. } | MetaOp::ResolveTxn { plans, .. } - | MetaOp::RecordCalvinWriteVersions { plans, .. } | MetaOp::CalvinExecuteStatic { plans, .. } | MetaOp::CalvinExecuteActive { plans, .. }, ) => plans.is_empty(), diff --git a/nodedb/src/control/server/shared/ddl/sqlstate.rs b/nodedb/src/control/server/shared/ddl/sqlstate.rs index 414c22ad9..e18b39213 100644 --- a/nodedb/src/control/server/shared/ddl/sqlstate.rs +++ b/nodedb/src/control/server/shared/ddl/sqlstate.rs @@ -69,6 +69,13 @@ pub fn error_code_to_sqlstate(code: &ErrorCode) -> (&'static str, &'static str, sqlstate::SERIALIZATION_FAILURE, format!("write refused without applying, retry: {reason}"), ), + // The fail-stopped core applied nothing, and another replica or a + // restart serves the statement: the class drivers retry on. + ErrorCode::CoreFailStopped { core_id, detail } => ( + "ERROR", + sqlstate::SERIALIZATION_FAILURE, + format!("core {core_id} is fail-stopped and applied nothing, retry: {detail}"), + ), ErrorCode::NotFound => ("ERROR", sqlstate::NO_DATA, "not found".into()), // `resource` is what makes the denial actionable: it says whether a // row-level-security policy refused the row or a grant is missing, and diff --git a/nodedb/src/control/server/shared/returning/inject.rs b/nodedb/src/control/server/shared/returning/inject.rs index 6b3ea3fba..9f1214ed4 100644 --- a/nodedb/src/control/server/shared/returning/inject.rs +++ b/nodedb/src/control/server/shared/returning/inject.rs @@ -413,7 +413,6 @@ pub fn inject_returning_spec(plan: &mut PhysicalPlan, spec: ReturningSpec) { | MetaOp::DropTxnOverlay { .. } | MetaOp::MarkSavepoint { .. } | MetaOp::RollbackToSavepoint { .. } - | MetaOp::RecordCalvinWriteVersions { .. } | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. } | MetaOp::ResolveTxn { .. } diff --git a/nodedb/src/control/server/shared/write_admission/admission_keys.rs b/nodedb/src/control/server/shared/write_admission/admission_keys.rs index 768d6424f..7d3b951fb 100644 --- a/nodedb/src/control/server/shared/write_admission/admission_keys.rs +++ b/nodedb/src/control/server/shared/write_admission/admission_keys.rs @@ -104,7 +104,6 @@ pub(super) fn is_calvin_apply(plan: &PhysicalPlan) -> bool { MetaOp::CalvinExecuteStatic { .. } | MetaOp::CalvinExecutePassive { .. } | MetaOp::CalvinExecuteActive { .. } - | MetaOp::RecordCalvinWriteVersions { .. } | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. } | MetaOp::CalvinResolve { .. } diff --git a/nodedb/src/control/server/shared/write_admission/leader_gate.rs b/nodedb/src/control/server/shared/write_admission/leader_gate.rs index 094b8bf09..e578b25e5 100644 --- a/nodedb/src/control/server/shared/write_admission/leader_gate.rs +++ b/nodedb/src/control/server/shared/write_admission/leader_gate.rs @@ -189,9 +189,11 @@ fn entry_scope(write: &ReplicatedWrite, database_id: crate::types::DatabaseId) - // scheduler holds its locks. A session transaction's redo writes // every collection it names. ReplicatedWrite::TransactionRedo { - body, collections, .. + collections, + calvin, + .. } => { - if body.is_calvin() { + if calvin.is_some() { EntryScope::Exempt } else { EntryScope::Collections(collections.clone()) diff --git a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs index e46c854a0..70fd4a5fc 100644 --- a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs +++ b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs @@ -359,7 +359,6 @@ pub fn plan_requires_txn_buffering(plan: &PhysicalPlan) -> bool { | MetaOp::DropTxnOverlay { .. } | MetaOp::MarkSavepoint { .. } | MetaOp::RollbackToSavepoint { .. } - | MetaOp::RecordCalvinWriteVersions { .. } | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. } | MetaOp::CalvinResolve { .. } @@ -1916,7 +1915,6 @@ mod tests { tenant_id: tenant(), plans: Vec::new(), epoch_system_ms: 0, - is_group_leader: false, versioned_reads: vec![VersionedReadEntry { engine: EngineTag::Kv, collection: QualifiedCollection::new(DatabaseId::DEFAULT, "c").to_string(), @@ -1945,7 +1943,6 @@ mod tests { plans: Vec::new(), injected_reads: BTreeMap::new(), epoch_system_ms: 0, - is_group_leader: false, }), PhysicalPlan::Meta(MetaOp::RebuildIndex { collection: QualifiedCollection::new(DatabaseId::DEFAULT, "c"), @@ -1974,10 +1971,6 @@ mod tests { txn_id: TxnId::new(1), savepoint: 1, }), - PhysicalPlan::Meta(MetaOp::RecordCalvinWriteVersions { - tenant_id: tenant(), - plans: Vec::new(), - }), PhysicalPlan::Meta(MetaOp::CalvinFlush { epoch: 0, position: 0, diff --git a/nodedb/src/control/server/shared/write_admission/predicate/user_data_write.rs b/nodedb/src/control/server/shared/write_admission/predicate/user_data_write.rs index 6daa3787b..e0e510a74 100644 --- a/nodedb/src/control/server/shared/write_admission/predicate/user_data_write.rs +++ b/nodedb/src/control/server/shared/write_admission/predicate/user_data_write.rs @@ -9,14 +9,26 @@ //! of a backup that already holds every row the tenant wrote. use crate::bridge::envelope::PhysicalPlan; -use nodedb_physical::physical_plan::{CrdtOp, KvOp}; +use nodedb_physical::physical_plan::{CrdtOp, KvOp, MetaOp}; use super::plan_is_write::plan_is_write; /// Whether `plan` writes a tenant's rows: a write-class plan that is not a /// schema install. pub fn plan_writes_user_data(plan: &PhysicalPlan) -> bool { - plan_is_write(plan) && !installs_schema(plan) + plan_is_write(plan) && !installs_schema(plan) && !calvin_install_without_user_write(plan) +} + +/// The install of a Calvin slice that raises no write mark: a slice that +/// writes only derived rows or schema. +fn calvin_install_without_user_write(plan: &PhysicalPlan) -> bool { + matches!( + plan, + PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { + calvin: Some(install), + .. + }) if !install.user_write + ) } /// A write-class plan that installs or removes schema state on a replica and @@ -73,4 +85,28 @@ mod tests { }); assert!(plan_writes_user_data(&plan)); } + + fn calvin_install(user_write: bool) -> PhysicalPlan { + PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { + redo: Vec::new(), + collections: vec!["c".into()], + sum_targets: Vec::new(), + origin: nodedb_physical::physical_plan::RedoOrigin::Commit, + calvin: Some(nodedb_physical::physical_plan::CalvinInstall { + epoch: 1, + position: 0, + epoch_system_ms: 0, + reply: nodedb_physical::physical_plan::CalvinReplySpec::Count(Vec::new()), + user_write, + }), + }) + } + + /// A Calvin slice that only installs schema, such as a constraint set + /// the leader's write gate sequenced, raises no write mark at install. + #[test] + fn a_calvin_install_counts_only_when_its_slice_writes_user_rows() { + assert!(plan_writes_user_data(&calvin_install(true))); + assert!(!plan_writes_user_data(&calvin_install(false))); + } } diff --git a/nodedb/src/control/server/sync/async_dispatch/delta/compensation.rs b/nodedb/src/control/server/sync/async_dispatch/delta/compensation.rs index 7fc11a1b9..6029411a7 100644 --- a/nodedb/src/control/server/sync/async_dispatch/delta/compensation.rs +++ b/nodedb/src/control/server/sync/async_dispatch/delta/compensation.rs @@ -195,6 +195,7 @@ fn compensation_hint_for_code(code: &ErrorCode) -> CompensationHint { | ErrorCode::CrdtFrontierMismatch { .. } | ErrorCode::CollectionDraining { .. } | ErrorCode::RetryableRefusal { .. } + | ErrorCode::CoreFailStopped { .. } | ErrorCode::TransactionRollback { .. } => CompensationHint::Retry { retry_after_ms: 0 }, other @ (ErrorCode::SyncRejected { .. } | ErrorCode::SyncNotApplied { .. } diff --git a/nodedb/src/control/server/sync/refusal.rs b/nodedb/src/control/server/sync/refusal.rs index ecf8e3d96..e97ccf6d8 100644 --- a/nodedb/src/control/server/sync/refusal.rs +++ b/nodedb/src/control/server/sync/refusal.rs @@ -25,6 +25,8 @@ pub(super) fn retryable_refusal_reason(error: &crate::Error) -> Option<&str> { crate::Error::RetryableRefusal { reason } => Some(reason), crate::Error::DataPlane(code) => match code { ErrorCode::RetryableRefusal { reason } => Some(reason), + // The fail-stopped core applied nothing: a restart serves the frame. + ErrorCode::CoreFailStopped { detail, .. } => Some(detail), ErrorCode::DeadlineExceeded | ErrorCode::RejectedConstraint { .. } | ErrorCode::RejectedPrevalidation { .. } @@ -353,6 +355,7 @@ fn is_indeterminate_code(code: &ErrorCode) -> bool { | ErrorCode::DispatchCapacity { .. } | ErrorCode::ConflictRetry | ErrorCode::RetryableRefusal { .. } + | ErrorCode::CoreFailStopped { .. } | ErrorCode::OllpRetryRequired | ErrorCode::CrdtFrontierMismatch { .. } | ErrorCode::CollectionDraining { .. } diff --git a/nodedb/src/control/state/calvin_counters.rs b/nodedb/src/control/state/calvin_counters.rs index 4eb519b05..c959c6df6 100644 --- a/nodedb/src/control/state/calvin_counters.rs +++ b/nodedb/src/control/state/calvin_counters.rs @@ -6,20 +6,20 @@ use std::sync::Arc; use std::sync::atomic::AtomicU64; /// Node-global Calvin observability counters, incremented by the per-vShard -/// schedulers (which hold `Arc`) and read by tests and metrics +/// schedulers and the data-group apply loops, and read by tests and metrics /// without reaching into the `!Send` Data-Plane state. All start at 0 and stay /// 0 in single-node / no-Calvin deployments. pub struct CalvinCounters { - /// Count of committed Calvin applies whose write versions were recorded - /// into the per-core write-version index, incremented once per apply for - /// which a scheduler dispatched a write-version record op. + /// Count of committed Calvin slices whose stamped redo installed on this + /// node. The install records the slice's write versions into the + /// per-core write-version index. pub write_versions_recorded: Arc, /// Count of committed Calvin applies whose participant reported that its /// slice of the transaction's reads was no longer current at apply time. /// Observation only — the apply still committed. pub read_set_validation_failures: Arc, - /// Count of staged Calvin transactions the scheduler resolved to COMMIT - /// by dispatching a flush of their commit-pending buffer to base. + /// Count of staged Calvin transactions the scheduler resolved to COMMIT. + /// Each commit's stamped redo installs its staged writes. pub commits_flushed: Arc, /// Count of staged Calvin transactions the scheduler resolved to ABORT /// by dispatching a drop of their commit-pending buffer, mirroring diff --git a/nodedb/src/control/state/calvin_local.rs b/nodedb/src/control/state/calvin_local.rs index 723f7b137..39dd4588a 100644 --- a/nodedb/src/control/state/calvin_local.rs +++ b/nodedb/src/control/state/calvin_local.rs @@ -10,7 +10,7 @@ use std::sync::{Arc, Mutex, OnceLock}; use crate::control::cluster::calvin::scheduler::lock::HotKeyTable; use crate::control::cluster::calvin::scheduler::lock_manager::{LockManager, TxnId}; use crate::control::cluster::calvin::scheduler::{ - CalvinAppliedLedgers, CaughtUpRegistry, SequencerProposer, + CalvinAppliedLedgers, CalvinInboxes, CaughtUpRegistry, SequencerProposer, }; use crate::control::server::shared::write_admission::AdmissionHolds; @@ -92,6 +92,9 @@ pub struct CalvinLocalState { /// Whether each running scheduler reached its rebuild target. Startup /// reads it before it opens the client gateway. pub caught_up: CaughtUpRegistry, + /// The inbox of each running scheduler: the data-group apply loop tells + /// it how each stamped Calvin redo concluded. + pub inboxes: CalvinInboxes, } impl CalvinLocalState { @@ -116,6 +119,7 @@ impl CalvinLocalState { sequencer_proposer: OnceLock::new(), applied: CalvinAppliedLedgers::default(), caught_up: CaughtUpRegistry::default(), + inboxes: CalvinInboxes::default(), } } } diff --git a/nodedb/src/control/wal_replication/decode/transaction_redo.rs b/nodedb/src/control/wal_replication/decode/transaction_redo.rs index 4e489728e..43c7a0ac5 100644 --- a/nodedb/src/control/wal_replication/decode/transaction_redo.rs +++ b/nodedb/src/control/wal_replication/decode/transaction_redo.rs @@ -11,7 +11,9 @@ use nodedb_physical::physical_plan::RedoOrigin; use nodedb_types::Surrogate; use super::super::transaction_redo::TransactionRedoPayload; -use super::super::types::{RedoBody, RedoContent, ReplicatedEventSource, ReplicatedWrite}; +use super::super::types::{ + CalvinRedoMeta, RedoBody, RedoContent, ReplicatedEventSource, ReplicatedWrite, +}; use crate::control::surrogate::CarriedIdentity; use crate::wal::RedoStreamId; @@ -36,18 +38,30 @@ pub struct ChunkedRedo { collections: Vec, event_source: ReplicatedEventSource, origin: RedoOrigin, + calvin: Option, } impl ChunkedRedo { /// The payload of the stream's assembled `content`. pub fn payload(&self, content: RedoContent) -> TransactionRedoPayload { - payload_of(content, &self.collections, self.event_source, self.origin) + payload_of( + content, + &self.collections, + self.event_source, + self.origin, + self.calvin.clone(), + ) } /// Every collection the transaction wrote. pub fn collections(&self) -> &[String] { &self.collections } + + /// The Calvin meta of a committed Calvin slice's redo. + pub fn calvin(&self) -> Option<&CalvinRedoMeta> { + self.calvin.as_ref() + } } fn payload_of( @@ -55,6 +69,7 @@ fn payload_of( collections: &[String], event_source: ReplicatedEventSource, origin: RedoOrigin, + calvin: Option, ) -> TransactionRedoPayload { TransactionRedoPayload { redo: content.redo, @@ -71,8 +86,7 @@ fn payload_of( .collect(), event_source: event_source.into(), origin, - // The wire entry carries no Calvin meta. - calvin: None, + calvin, } } @@ -83,6 +97,7 @@ pub fn decode_transaction_redo(write: &ReplicatedWrite) -> crate::Result crate::Result DecodedTransactionRedo::Chunked(ChunkedRedo { stream: *stream, @@ -104,6 +120,7 @@ pub fn decode_transaction_redo(write: &ReplicatedWrite) -> crate::Result crate::Result { + redo_proposal(target, payload, max_entry_bytes, |last| { + RedoStreamId::Session { + vshard: target.vshard_id.as_u32(), + idempotency_key: last.idempotency_key, + } + }) +} + +/// What the leader stamps on a committed Calvin slice's final entry. Every +/// re-proposal of the slice carries the same stamps. +#[derive(Debug, Clone)] +pub struct CalvinEntryStamps { + /// The slice's commit HLC: every replica records it on the tenant's + /// write mark. + pub write_hlc: u64, + /// The RESTORE that re-issued the slice, `0` for any other. + pub restore_id: u64, + /// The metadata index the coordinator planned the transaction against. + pub metadata_floor: u64, + /// The incarnation the coordinator planned against for each collection + /// the slice writes. + pub incarnations: Vec, +} + +/// The entries of a committed Calvin slice's redo, for one proposal +/// attempt. The redo travels inline when its entry fits `max_entry_bytes`, +/// else chunked under `stream`. +pub fn calvin_redo_proposal( + target: RedoEntryTarget, + payload: &TransactionRedoPayload, + stamps: &CalvinEntryStamps, + stream: RedoStreamId, + max_entry_bytes: usize, +) -> crate::Result { + let proposal = redo_proposal(target, payload, max_entry_bytes, |_| stream)?; + let stamp = |mut last: ReplicatedEntry| { + last.write_hlc = stamps.write_hlc; + last.restore_id = stamps.restore_id; + last.metadata_floor = stamps.metadata_floor; + last.incarnations = stamps.incarnations.clone(); + last + }; + Ok(match proposal { + RedoProposal::Inline(last) => RedoProposal::Inline(stamp(last)), + RedoProposal::Chunked { + stream, + chunks, + last, + } => RedoProposal::Chunked { + stream, + chunks, + last: stamp(last), + }, + }) +} + +/// The entries of `payload`: inline when its entry fits `max_entry_bytes`, +/// else chunks under the stream `stream_of` names for the final entry. +fn redo_proposal( + target: RedoEntryTarget, + payload: &TransactionRedoPayload, + max_entry_bytes: usize, + stream_of: impl FnOnce(&ReplicatedEntry) -> RedoStreamId, ) -> crate::Result { let content = content_of(payload); let bytes = content.to_bytes()?; @@ -126,10 +191,7 @@ pub fn session_redo_proposal( if bytes.len().saturating_add(envelope) <= max_entry_bytes { return Ok(RedoProposal::Inline(last)); } - let stream = RedoStreamId::Session { - vshard: target.vshard_id.as_u32(), - idempotency_key: last.idempotency_key, - }; + let stream = stream_of(&last); let chunks = chunk_entries( target, stream, @@ -152,6 +214,7 @@ pub fn session_redo_proposal( collections: payload.collections.clone(), event_source: ReplicatedEventSource::from(payload.event_source), origin: payload.origin, + calvin: payload.calvin.clone(), }; Ok(RedoProposal::Chunked { stream, @@ -246,6 +309,95 @@ mod tests { } } + /// The payload of a committed Calvin slice. + fn calvin_payload(op_bytes: usize) -> TransactionRedoPayload { + let mut payload = payload(op_bytes); + payload.calvin = Some(super::super::super::types::CalvinRedoMeta { + epoch_system_ms: 7, + reply: nodedb_physical::physical_plan::CalvinReplySpec::Count(Vec::new()), + primary_write: true, + user_write: true, + returning: false, + }); + payload + } + + fn calvin_stamps() -> CalvinEntryStamps { + CalvinEntryStamps { + write_hlc: 41, + restore_id: 0, + metadata_floor: 9, + incarnations: vec![CollectionIncarnation { + collection: "orders".into(), + incarnation: nodedb_types::Hlc::ZERO, + }], + } + } + + fn calvin_stream() -> RedoStreamId { + RedoStreamId::Calvin { + vshard: 3, + epoch: 5, + position: 1, + attempt: 2, + } + } + + /// The final entry of a Calvin slice's redo carries the leader's stamps + /// and the slice's meta, inline or chunked. Chunks carry no stamp. + #[test] + fn a_calvin_proposal_stamps_its_final_entry() { + let inline = calvin_redo_proposal( + target(), + &calvin_payload(100), + &calvin_stamps(), + calvin_stream(), + 64 * 1024, + ) + .expect("build"); + let RedoProposal::Inline(last) = inline else { + panic!("a small redo travels inline"); + }; + assert_eq!(last.write_hlc, 41); + assert_eq!(last.metadata_floor, 9); + assert_eq!(last.incarnations, calvin_stamps().incarnations); + assert!(matches!( + last.write, + ReplicatedWrite::TransactionRedo { + calvin: Some(_), + .. + } + )); + + let chunked = calvin_redo_proposal( + target(), + &calvin_payload(300_000), + &calvin_stamps(), + calvin_stream(), + 64 * 1024, + ) + .expect("build"); + let RedoProposal::Chunked { + stream, + chunks, + last, + } = chunked + else { + panic!("a 300 KB redo travels chunked"); + }; + assert_eq!(stream, calvin_stream(), "the attempt names its stream"); + assert_eq!(last.write_hlc, 41); + assert!(chunks.iter().all(|chunk| chunk.write_hlc == 0)); + assert!(matches!( + last.write, + ReplicatedWrite::TransactionRedo { + body: RedoBody::Chunked { stream: named, .. }, + calvin: Some(_), + .. + } if named == calvin_stream() + )); + } + #[test] fn a_redo_within_the_limit_travels_inline() { let proposal = session_redo_proposal(target(), &payload(100), 64 * 1024).expect("build"); diff --git a/nodedb/src/control/wal_replication/transaction_redo/chunks/store.rs b/nodedb/src/control/wal_replication/transaction_redo/chunks/store.rs index 8919cd4aa..5bdcef794 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/chunks/store.rs +++ b/nodedb/src/control/wal_replication/transaction_redo/chunks/store.rs @@ -361,6 +361,24 @@ impl RedoChunkStore { }); self.note_open(&state); } + + /// Drop every Calvin stream whose position `applied` names. Boot calls + /// it once the applied ledgers recovered: a position that applied + /// through one copy installs no other. + pub fn drop_applied_calvin_streams(&self, applied: impl Fn(u32, u64, u32) -> bool) { + if self.open_streams() == 0 { + return; + } + let mut state = self.lock(); + state.streams.retain(|stream, _| { + !matches!( + stream, + RedoStreamId::Calvin { vshard, epoch, position, .. } + if applied(*vshard, *epoch, *position) + ) + }); + self.note_open(&state); + } } #[cfg(test)] @@ -543,4 +561,37 @@ pub(super) mod tests { store.drop_calvin_position(2, 5, 1); assert_eq!(store.open_streams(), 0); } + + /// Boot drops the streams of every position its ledgers hold applied, + /// and keeps the rest. + #[tokio::test] + async fn boot_drops_the_streams_of_applied_calvin_positions() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RedoChunkStore::new(wal(&dir), limits(1 << 20)); + let calvin = |position| RedoStreamId::Calvin { + vshard: 2, + epoch: 5, + position, + attempt: 0, + }; + for position in [1, 2] { + store + .apply_chunk(chunk(calvin(position), 0, 2, b"ab")) + .await + .expect("chunk"); + } + store + .apply_chunk(chunk(session(7), 0, 2, b"ab")) + .await + .expect("chunk"); + + store.drop_applied_calvin_streams(|vshard, epoch, position| { + (vshard, epoch, position) == (2, 5, 1) + }); + + assert_eq!(store.open_streams(), 2); + assert!(store.take_for_final(&calvin(1)).is_none()); + assert!(store.take_for_final(&calvin(2)).is_some()); + assert!(store.take_for_final(&session(7)).is_some()); + } } diff --git a/nodedb/src/control/wal_replication/transaction_redo/mod.rs b/nodedb/src/control/wal_replication/transaction_redo/mod.rs index 883e9f801..62a4091fd 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/mod.rs +++ b/nodedb/src/control/wal_replication/transaction_redo/mod.rs @@ -22,5 +22,5 @@ pub(crate) use abandon::spawn_redo_abandoner; pub use apply::RedoTarget; pub(crate) use apply::{enqueue_transaction_redo, record_cross_shard_key}; pub use chunks::{RedoChunkError, RedoChunkStore}; -pub use payload::TransactionRedoPayload; +pub use payload::{CalvinSlice, TransactionRedoPayload}; pub(crate) use propose::propose_transaction_redo; diff --git a/nodedb/src/control/wal_replication/transaction_redo/payload.rs b/nodedb/src/control/wal_replication/transaction_redo/payload.rs index 4c99e5342..0f8796fa6 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/payload.rs +++ b/nodedb/src/control/wal_replication/transaction_redo/payload.rs @@ -38,6 +38,18 @@ pub struct TransactionRedoPayload { pub calvin: Option, } +/// What a committed Calvin slice's redo carries beside its record, derived +/// from the slice's local plans when it stages. +#[derive(Debug, Clone)] +pub struct CalvinSlice { + pub collections: Vec, + pub sum_targets: Vec, + pub identities: Vec, + pub event_source: EventSource, + pub origin: RedoOrigin, + pub meta: CalvinRedoMeta, +} + impl TransactionRedoPayload { /// The payload for a commit whose buffered write plans are `plans` and /// whose resolved redo is `redo`, built on the node that resolved it. @@ -65,6 +77,29 @@ impl TransactionRedoPayload { }) } + /// The payload of a committed Calvin slice whose resolved redo is `redo`, + /// built on the vShard's data-group leader. The record's `calvin_stamp` + /// must name the slice's `(epoch, position)`. + pub fn from_calvin(redo: RedoRecord, slice: CalvinSlice) -> Self { + let CalvinSlice { + collections, + sum_targets, + identities, + event_source, + origin, + meta, + } = slice; + Self { + redo, + collections, + sum_targets, + identities, + event_source, + origin, + calvin: Some(meta), + } + } + /// The Data-Plane plan that applies this redo. pub fn apply_plan(&self) -> crate::Result { Ok(PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { @@ -97,6 +132,7 @@ impl TransactionRedoPayload { position: stamp.position, epoch_system_ms: meta.epoch_system_ms, reply: meta.reply.clone(), + user_write: meta.user_write, })) } } @@ -135,6 +171,8 @@ mod tests { epoch_system_ms: 1_700, reply: CalvinReplySpec::Count(vec![7]), primary_write: true, + user_write: true, + returning: false, } } @@ -157,8 +195,6 @@ mod tests { epoch: 9, position: 4, vshard_id: 2, - collections: Vec::new(), - sum_targets: Vec::new(), }; let calvin = payload(Some(stamp), Some(meta())); assert_eq!( @@ -168,6 +204,7 @@ mod tests { position: 4, epoch_system_ms: 1_700, reply: CalvinReplySpec::Count(vec![7]), + user_write: true, }) ); } diff --git a/nodedb/src/control/wal_replication/types/redo_chunk_wire.rs b/nodedb/src/control/wal_replication/types/redo_chunk_wire.rs index 3db3aa184..1687fc269 100644 --- a/nodedb/src/control/wal_replication/types/redo_chunk_wire.rs +++ b/nodedb/src/control/wal_replication/types/redo_chunk_wire.rs @@ -63,14 +63,3 @@ pub enum RedoBody { len: u64, }, } - -impl RedoBody { - /// Whether the body is a committed Calvin slice's. Its scheduler holds - /// its locks. - pub fn is_calvin(&self) -> bool { - match self { - Self::Inline(content) => content.redo.calvin_stamp.is_some(), - Self::Chunked { stream, .. } => matches!(stream, RedoStreamId::Calvin { .. }), - } - } -} diff --git a/nodedb/src/control/wal_replication/types/replicated_write.rs b/nodedb/src/control/wal_replication/types/replicated_write.rs index 5c6067f54..87ae3742f 100644 --- a/nodedb/src/control/wal_replication/types/replicated_write.rs +++ b/nodedb/src/control/wal_replication/types/replicated_write.rs @@ -1034,9 +1034,8 @@ pub enum ReplicatedWrite { /// Every replica, the proposer included, appends the body's redo to its /// own WAL and applies it through the WAL replay arms, in Raft log order. /// The record is resolved once, on the proposer; no replica re-derives - /// it. The redo's `calvin_stamp` names the Calvin `(epoch, position)` the - /// record applies, when a Calvin slice produced it. A chunked body - /// assembles from the `RedoChunk` entries of its stream. + /// it. A chunked body assembles from the `RedoChunk` entries of its + /// stream. TransactionRedo { body: RedoBody, /// Every collection the transaction wrote, for the collection-floor @@ -1045,6 +1044,9 @@ pub enum ReplicatedWrite { event_source: ReplicatedEventSource, /// Which commit-boundary checks every replica's apply runs. origin: nodedb_physical::physical_plan::RedoOrigin, + /// Set for a committed Calvin slice. The redo's `calvin_stamp` names + /// its `(epoch, position)`, and the slice's scheduler holds its locks. + calvin: Option, }, /// A backup's consistent cut through this group's log. It writes no /// data. Every entry before it applies before the backup snapshots, and diff --git a/nodedb/src/control/wal_replication/types/transaction_redo_wire.rs b/nodedb/src/control/wal_replication/types/transaction_redo_wire.rs index 1edd53a5d..8b715617a 100644 --- a/nodedb/src/control/wal_replication/types/transaction_redo_wire.rs +++ b/nodedb/src/control/wal_replication/types/transaction_redo_wire.rs @@ -24,8 +24,15 @@ pub struct CalvinRedoMeta { /// its install. pub reply: CalvinReplySpec, /// Whether the slice writes a row the statement names, not only a - /// derived row. + /// derived row. Only a primary slice deposits the statement's reply. pub primary_write: bool, + /// Whether the install raises the tenant's write mark: a primary write + /// that changes a tenant row. A slice that only installs schema raises + /// none. + pub user_write: bool, + /// Whether the slice answers rows (a RETURNING write), not a count. + /// Every replica reads it to build the same `CompletionAck` result. + pub returning: bool, } /// One `(collection, primary key) → surrogate` identity a committed diff --git a/nodedb/src/data/executor/core_loop/calvin_fence.rs b/nodedb/src/data/executor/core_loop/calvin_fence.rs deleted file mode 100644 index d6d49dca4..000000000 --- a/nodedb/src/data/executor/core_loop/calvin_fence.rs +++ /dev/null @@ -1,371 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Calvin key ownership on a core. -//! -//! A staged Calvin transaction owns the rows it staged from its stage until -//! its flush or drop. Its flush installs images the stage computed, so a -//! write that lands on one of those rows in between would be overwritten. -//! A write to an owned row therefore waits here, in arrival order. -//! -//! - A document or KV point write waits only when a staged transaction staged -//! its row, or truncated its collection. -//! - Every other write, and a committed session transaction's redo, waits -//! when a staged transaction writes its collection. -//! - A CRDT constraint-set install writes no row and never waits. -//! -//! When the owner resolves, each waiting write that no other owner fences -//! runs, in arrival order. A write whose WAL record sits below the redo record -//! of a flush it waited on is refused with `RetryableRefusal`: restart replay -//! applies records in LSN order, so it would install that write before the -//! flush. The refusal cancels its record, and the client retries. A write -//! still waiting at its request deadline is refused the same way. - -use std::collections::VecDeque; -use std::time::Instant; - -use nodedb_physical::physical_plan::{CrdtOp, DocumentOp, KvOp, MetaOp, PhysicalPlan}; -use nodedb_types::{RowIdentity, TenantId}; - -use super::CoreLoop; -use crate::bridge::envelope::{ErrorCode, Response}; -use crate::control::server::shared::write_admission::plan_is_write; -use crate::control::wal_replication::transaction_redo::collections::written_collections; -use crate::data::executor::handlers::control::calvin_synthetic_txn_id; -use crate::data::executor::handlers::transaction::stage_write::kv_row_identity; -use crate::data::executor::task::ExecutionTask; -use crate::types::Lsn; - -/// `(epoch, position, vshard)` of a staged Calvin transaction. -type Owner = (u64, u32, u32); - -/// A write waiting for the Calvin transaction that owns its rows. -pub(in crate::data::executor) struct ParkedWrite { - task: ExecutionTask, - owner: Owner, - /// The highest redo LSN among the flushes this write waited on. - passed_flush_lsn: Option, -} - -/// The writes waiting on this core, in arrival order, and the owners that -/// resolved since the core last released writes. -#[derive(Default)] -pub(in crate::data::executor) struct CalvinFence { - parked: VecDeque, - /// Each owner that flushed, with its redo LSN, or dropped, with `None`. - resolved: Vec<(Owner, Option)>, -} - -#[cfg(test)] -impl CalvinFence { - /// Number of waiting writes. - pub(in crate::data::executor) fn len(&self) -> usize { - self.parked.len() - } -} - -impl CalvinFence { - /// Record that the staged transaction `owner` flushed at `flush_lsn`, or - /// dropped when `flush_lsn` is `None`. The core releases its waiting - /// writes once the resolving request answered. - pub(in crate::data::executor) fn note_resolved( - &mut self, - owner: (u64, u32, u32), - flush_lsn: Option, - ) { - if !self.parked.is_empty() { - self.resolved.push((owner, flush_lsn)); - } - } -} - -/// What a write targets, as the fence compares it with staged state. -enum FenceTarget { - /// Every row of a collection. - Collection(String), - /// One document row, by surrogate. - Surrogate(String, u32), - /// One KV row, by its overlay identity. - Row(String, RowIdentity), -} - -impl FenceTarget { - fn collection(&self) -> &str { - match self { - Self::Collection(collection) - | Self::Surrogate(collection, _) - | Self::Row(collection, _) => collection, - } - } -} - -/// The targets of `plan`, or `None` when it writes no base state. -fn fence_targets(plan: &PhysicalPlan) -> Option> { - if let PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { collections, .. }) = plan { - return Some( - collections - .iter() - .cloned() - .map(FenceTarget::Collection) - .collect(), - ); - } - // A constraint-set install replaces the per-core CRDT validator's set for - // its collection, fenced by its own constraint version. It writes no row, - // so no flush can overwrite it. Parking it holds its data group's - // apply loop, and every later write in that group, behind the flush. - if matches!( - plan, - PhysicalPlan::Meta( - MetaOp::CalvinExecuteStatic { .. } - | MetaOp::CalvinExecuteActive { .. } - | MetaOp::CalvinFlush { .. } - | MetaOp::CalvinDrop { .. } - | MetaOp::CalvinResolve { .. } - ) | PhysicalPlan::Crdt(CrdtOp::SetConstraints { .. } | CrdtOp::DropConstraints { .. }) - ) || !plan_is_write(plan) - { - return None; - } - let point = match plan { - PhysicalPlan::Document( - DocumentOp::PointPut { - collection, - surrogate, - .. - } - | DocumentOp::PointInsert { - collection, - surrogate, - .. - }, - ) => vec![FenceTarget::Surrogate( - collection.as_str().to_string(), - surrogate.as_u32(), - )], - // A key unbound in this database names no row to fence. - PhysicalPlan::Document( - DocumentOp::PointUpdate { - collection, - surrogate, - .. - } - | DocumentOp::PointDelete { - collection, - surrogate, - .. - }, - ) => surrogate - .iter() - .map(|s| FenceTarget::Surrogate(collection.as_str().to_string(), s.as_u32())) - .collect(), - PhysicalPlan::Kv( - KvOp::Put { - collection, key, .. - } - | KvOp::Insert { - collection, key, .. - } - | KvOp::InsertIfAbsent { - collection, key, .. - } - | KvOp::Expire { - collection, key, .. - } - | KvOp::Persist { - collection, key, .. - }, - ) => vec![FenceTarget::Row( - collection.as_str().to_string(), - kv_row_identity(key), - )], - PhysicalPlan::Kv(KvOp::Delete { - collection, keys, .. - }) => keys - .iter() - .map(|key| FenceTarget::Row(collection.as_str().to_string(), kv_row_identity(key))) - .collect(), - _ => written_collections(std::slice::from_ref(plan)) - .into_iter() - .map(FenceTarget::Collection) - .collect(), - }; - Some(point) -} - -impl CoreLoop { - /// Hold `task` when a staged Calvin transaction owns a row it writes. - /// Returns the task when nothing owns its rows. - pub(in crate::data::executor) fn park_if_calvin_owned( - &mut self, - task: ExecutionTask, - ) -> Option { - match self.calvin_owner_of(&task) { - Some(owner) => { - // Raft fixed this write's order. Parking it holds its data - // group's applied index behind the owner's flush, so the - // capture names the write and its owner. - if matches!( - task.request.admission, - crate::bridge::envelope::Admission::Exempt( - crate::bridge::envelope::ExemptReason::AlreadyOrdered - ) - ) { - crate::diag::replicated_write_parked( - self.core_id, - task.plan() - .named_collections() - .first() - .copied() - .unwrap_or(""), - owner, - ); - } - self.calvin.fence.parked.push_back(ParkedWrite { - task, - owner, - passed_flush_lsn: None, - }); - None - } - None => Some(task), - } - } - - /// The earliest staged Calvin transaction that owns a row `task` writes. - fn calvin_owner_of(&self, task: &ExecutionTask) -> Option { - if self.calvin.commit_pending.is_empty() { - return None; - } - let targets = fence_targets(task.plan())?; - let database_id = task.request.database_id; - let tenant_id = task.request.tenant_id; - let mut owners: Vec = self.calvin.commit_pending.keys().copied().collect(); - owners.sort_unstable(); - owners - .into_iter() - .find(|owner| self.owns_any(*owner, database_id, tenant_id, &targets)) - } - - /// Whether the staged transaction `owner` owns any of `targets`. - fn owns_any( - &self, - owner: Owner, - database_id: crate::types::DatabaseId, - tenant_id: TenantId, - targets: &[FenceTarget], - ) -> bool { - let Some(pending) = self.calvin.commit_pending.get(&owner) else { - return false; - }; - if pending.tenant_id != tenant_id { - return false; - } - let written = written_collections(&pending.plans); - let overlay = calvin_synthetic_txn_id(owner.0, owner.1, owner.2) - .ok() - .and_then(|txn_id| self.txn_overlays.get(&txn_id)); - targets.iter().any(|target| { - if !written.iter().any(|c| c == target.collection()) { - return false; - } - let coll_key = (database_id, tenant_id, target.collection().to_string()); - match target { - FenceTarget::Collection(_) => true, - FenceTarget::Surrogate(_, surrogate) => overlay.is_none_or(|overlay| { - overlay.is_truncated(&coll_key) - || overlay.get(&coll_key, *surrogate).is_some() - || overlay.get_ttl(&coll_key, *surrogate).is_some() - }), - FenceTarget::Row(_, identity) => overlay.is_none_or(|overlay| { - overlay.is_truncated(&coll_key) - || overlay.surrogate_for_doc_id(&coll_key, identity).is_some() - }), - } - }) - } - - /// Run every waiting write no owner fences any more, once an owner - /// resolved. - pub(in crate::data::executor) fn release_resolved_calvin_owners(&mut self) { - let resolved = std::mem::take(&mut self.calvin.fence.resolved); - if resolved.is_empty() { - return; - } - for (owner, flush_lsn) in resolved { - let Some(lsn) = flush_lsn else { - continue; - }; - for parked in &mut self.calvin.fence.parked { - if parked.owner == owner { - parked.passed_flush_lsn = parked.passed_flush_lsn.max(Some(lsn)); - } - } - } - let parked = std::mem::take(&mut self.calvin.fence.parked); - for mut write in parked { - if let Some(next) = self.calvin_owner_of(&write.task) { - write.owner = next; - self.calvin.fence.parked.push_back(write); - continue; - } - let below_flush = write - .passed_flush_lsn - .is_some_and(|flush| write.task.wal_lsn().is_some_and(|lsn| lsn < flush)); - if below_flush { - let response = self.response_error( - &write.task, - ErrorCode::RetryableRefusal { - reason: "a Calvin transaction installed this row at a later log \ - position while the write waited" - .into(), - }, - ); - self.send_parked_response(response); - } else { - self.run_task(write.task); - } - } - } - - /// Refuse every waiting write whose request deadline passed. - pub(in crate::data::executor) fn expire_calvin_parked(&mut self) { - if self.calvin.fence.parked.is_empty() { - return; - } - // no-determinism: the request deadline bounds a wait; the refusal - // cancels the write's record, so no replica applies it. - let now = Instant::now(); - let parked = std::mem::take(&mut self.calvin.fence.parked); - for write in parked { - if now > write.task.request.deadline { - self.drop_journal_group(write.task.request_id().as_u64()); - let response = self.response_error( - &write.task, - ErrorCode::RetryableRefusal { - reason: format!( - "the write waited past its deadline for Calvin transaction \ - {}/{} that owns its row", - write.owner.0, write.owner.1 - ), - }, - ); - self.send_parked_response(response); - } else { - self.calvin.fence.parked.push_back(write); - } - } - } - - fn send_parked_response(&mut self, response: Response) { - if let Err(e) = self - .response_tx - .try_push(crate::bridge::dispatch::BridgeResponse { inner: response }) - { - tracing::warn!( - core = self.core_id, - error = %e, - "failed to send a parked write's refusal: response queue full" - ); - } - } -} diff --git a/nodedb/src/data/executor/core_loop/calvin_state.rs b/nodedb/src/data/executor/core_loop/calvin_state.rs index 212f96085..8e5b152f1 100644 --- a/nodedb/src/data/executor/core_loop/calvin_state.rs +++ b/nodedb/src/data/executor/core_loop/calvin_state.rs @@ -4,50 +4,29 @@ use std::collections::HashMap; -use super::calvin_fence::CalvinFence; use super::commit_pending::PendingCommit; -/// Staged Calvin transactions, the writes that wait on them, and the leader -/// flag of the transaction executing now. +/// Staged Calvin transactions this core holds. pub(in crate::data::executor) struct CalvinCoreState { /// Staged Calvin transactions awaiting the global verdict, keyed by /// `(epoch, position, vshard)`. `CalvinExecuteStatic` validates a /// transaction, stages its plans into the synthetic overlay, and inserts - /// its entry here without mutating base. The verdict-driven `CalvinFlush` - /// installs the transaction's redo record, and `CalvinDrop` discards the - /// staged state. Nothing staged here is observable in the base engines - /// until a flush. + /// its entry here without mutating base. The install of the slice's + /// stamped redo entry consumes the entry, and `CalvinDrop` discards it. + /// Nothing staged here is observable in the base engines until the + /// install. /// /// The vShard is part of the key because vShards round-robin onto cores. /// Several vShard slices of one multi-participant transaction share the /// same `(epoch, position)` and can land on the same core. Each slice - /// stages and flushes on its own. + /// stages and installs on its own. pub(in crate::data::executor) commit_pending: HashMap<(u64, u32, u32), PendingCommit>, - - /// Whether this node leads the data group of the Calvin transaction - /// executing now. - /// - /// `execute_calvin_execute_active` sets it from the scheduler-stamped, - /// per-node `is_group_leader` around its OLLP verification, then restores - /// the resting value `true`. A shard that runs a bulk DML directly has no - /// replication followers, so it must run OLLP drift verification. - /// - /// OLLP determinism: the verification emits `OllpRetryRequired` only when - /// this is `true`. Every replica stages the carried - /// `ollp_predicted_surrogates` set verbatim, so all replicas write the same - /// surrogate set whatever their local scans read. - pub(in crate::data::executor) ollp_is_group_leader: bool, - - /// Writes that wait for the staged transaction owning their rows. - pub(in crate::data::executor) fence: CalvinFence, } impl CalvinCoreState { pub(in crate::data::executor) fn new() -> Self { Self { commit_pending: HashMap::new(), - ollp_is_group_leader: true, - fence: CalvinFence::default(), } } } diff --git a/nodedb/src/data/executor/core_loop/commit_pending.rs b/nodedb/src/data/executor/core_loop/commit_pending.rs index 59f46f60b..0cc1abf45 100644 --- a/nodedb/src/data/executor/core_loop/commit_pending.rs +++ b/nodedb/src/data/executor/core_loop/commit_pending.rs @@ -7,14 +7,14 @@ use crate::types::TenantId; use nodedb_physical::physical_plan::PhysicalPlan; /// A Calvin transaction staged for commit, held between the -/// validate-and-stage step and the verdict-driven flush-or-drop. +/// validate-and-stage step and its install or drop. /// /// `CalvinExecuteStatic` and `CalvinExecuteActive` stage the plans into the /// synthetic overlay and insert this entry WITHOUT mutating base or firing /// side effects. `CalvinResolve` resolves the overlay into the transaction's -/// redo record. A verdict-driven `CalvinFlush` installs that record and -/// answers with `reply`. A `CalvinDrop` discards the entry. -/// Nothing here is observable in the base engines until a flush. +/// redo record. The install of the slice's stamped redo entry consumes the +/// entry and answers with `reply`. A `CalvinDrop` discards the entry. +/// Nothing here is observable in the base engines until the install. pub(in crate::data::executor) struct PendingCommit { /// The staged write plans `CalvinResolve` resolves against the overlay. pub plans: Vec, diff --git a/nodedb/src/data/executor/core_loop/fail_stop.rs b/nodedb/src/data/executor/core_loop/fail_stop.rs index 5f81ddcff..c3474732c 100644 --- a/nodedb/src/data/executor/core_loop/fail_stop.rs +++ b/nodedb/src/data/executor/core_loop/fail_stop.rs @@ -16,8 +16,9 @@ //! records the node-wide marker that //! `/healthz`, the native `STATUS` and the `nodedb_data_plane_core_fail_stopped` //! gauge read, and latches the core. A latched core refuses every request it -//! dequeues with `RetryableRefusal`: it applied nothing, so the funnel +//! dequeues with `CoreFailStopped`: it applied nothing, so the funnel //! cancels the request's record and another replica or a restart serves it. +//! A Calvin scheduler reads the code and halts instead of voting. //! The latch never clears. use tracing::{error, warn}; @@ -140,7 +141,7 @@ impl CoreLoop { let Some((cause, _)) = self.fail_stop.cause() else { return 0; }; - let reason = format!( + let detail = format!( "core {} is fail-stopped ({}): its state is unknown until restart", self.core_id, cause.label() @@ -150,8 +151,9 @@ impl CoreLoop { self.drop_journal_group(task.request_id().as_u64()); let response = self.response_error( &task, - ErrorCode::RetryableRefusal { - reason: reason.clone(), + ErrorCode::CoreFailStopped { + core_id: self.core_id, + detail: detail.clone(), }, ); if let Err(e) = self diff --git a/nodedb/src/data/executor/core_loop/mod.rs b/nodedb/src/data/executor/core_loop/mod.rs index cf3575947..afdbb8fcc 100644 --- a/nodedb/src/data/executor/core_loop/mod.rs +++ b/nodedb/src/data/executor/core_loop/mod.rs @@ -3,7 +3,6 @@ mod accessors; pub(in crate::data::executor) mod apply_scope; mod bitemporal_time; -pub(in crate::data::executor) mod calvin_fence; pub(in crate::data::executor) mod calvin_state; pub(in crate::data::executor) mod checkpoint_floors; mod columnar_schema_seed; diff --git a/nodedb/src/data/executor/core_loop/tick.rs b/nodedb/src/data/executor/core_loop/tick.rs index d894e9074..4c5739086 100644 --- a/nodedb/src/data/executor/core_loop/tick.rs +++ b/nodedb/src/data/executor/core_loop/tick.rs @@ -58,14 +58,10 @@ impl CoreLoop { }; self.io_metrics.record_wait(tier, wait_ns); - // A write to a row a staged Calvin transaction owns waits for it. // A plain REINDEX starts its rebuilds and waits for their cutovers. - if let Some(task) = self.park_if_calvin_owned(qt.task) - && let Some(task) = self.hold_plain_reindex(task) - { + if let Some(task) = self.hold_plain_reindex(qt.task) { self.run_task(task); } - self.release_resolved_calvin_owners(); true } @@ -184,7 +180,6 @@ impl CoreLoop { // Adjust SPSC read depth based on current memory pressure. self.apply_spsc_pressure(); self.drain_requests(); - self.expire_calvin_parked(); let mut processed = 0; while !self.task_queue.is_empty() { // A fail-stopped core serves nothing, including the rest of the @@ -337,7 +332,7 @@ mod tests { assert!( matches!( resp.inner.error_code.as_deref(), - Some(ErrorCode::RetryableRefusal { .. }) + Some(ErrorCode::CoreFailStopped { .. }) ), "{:?}", resp.inner.error_code diff --git a/nodedb/src/data/executor/core_loop/write_index.rs b/nodedb/src/data/executor/core_loop/write_index.rs index 2ac9f7d67..16bc04f08 100644 --- a/nodedb/src/data/executor/core_loop/write_index.rs +++ b/nodedb/src/data/executor/core_loop/write_index.rs @@ -131,6 +131,18 @@ impl WriteVersionIndex { self.coll_write_lsn.get(key).copied() } + /// Every recorded per-key version. + #[cfg(test)] + pub(crate) fn recorded_keys(&self) -> impl Iterator { + self.last_write_lsn.iter().map(|(key, lsn)| (key, *lsn)) + } + + /// Every recorded per-collection floor. + #[cfg(test)] + pub(crate) fn recorded_collections(&self) -> impl Iterator { + self.coll_write_lsn.iter().map(|(key, lsn)| (key, *lsn)) + } + /// Whether a previously observed read is still current against this /// core's recorded write versions. /// diff --git a/nodedb/src/data/executor/dispatch/meta.rs b/nodedb/src/data/executor/dispatch/meta.rs index d92529090..89e101470 100644 --- a/nodedb/src/data/executor/dispatch/meta.rs +++ b/nodedb/src/data/executor/dispatch/meta.rs @@ -167,7 +167,6 @@ impl CoreLoop { tenant_id, plans, epoch_system_ms, - is_group_leader, versioned_reads, body_plans, } => self.execute_calvin_execute_static( @@ -176,7 +175,6 @@ impl CoreLoop { epoch: *epoch, position: *position, epoch_system_ms: *epoch_system_ms, - is_group_leader: *is_group_leader, }, tenant_id, plans, @@ -204,14 +202,12 @@ impl CoreLoop { plans, injected_reads, epoch_system_ms, - is_group_leader, } => self.execute_calvin_execute_active( task, CalvinExecCtx { epoch: *epoch, position: *position, epoch_system_ms: *epoch_system_ms, - is_group_leader: *is_group_leader, }, tenant_id, plans, @@ -236,16 +232,6 @@ impl CoreLoop { self.execute_delete_synonym_group(task, *tenant_id, name) } - MetaOp::RecordCalvinWriteVersions { tenant_id, plans } => { - // The Calvin apply already committed; this records the write - // version of every key it wrote at the applied WAL LSN the - // scheduler threaded onto the request envelope. A no-op when - // the envelope carries no LSN. The install recorded the - // index-value versions of its document rows itself. - self.record_batch_write_versions(task, tenant_id.as_u64(), plans); - self.response_ok(task) - } - MetaOp::CalvinFlush { epoch, position, diff --git a/nodedb/src/data/executor/enforcement/materialized_sum/divergence.rs b/nodedb/src/data/executor/enforcement/materialized_sum/divergence.rs index f19fca434..35bbf3fa6 100644 --- a/nodedb/src/data/executor/enforcement/materialized_sum/divergence.rs +++ b/nodedb/src/data/executor/enforcement/materialized_sum/divergence.rs @@ -71,16 +71,13 @@ impl CoreLoop { /// require. /// /// `rows` are already-decoded row images. A collection that declares no - /// materialized-sum binding, and every replica that is not the group leader, - /// diverge vacuously — they return `false` without reading anything. + /// materialized-sum binding diverges vacuously: it returns `false` without + /// reading anything. pub(in crate::data::executor) fn sum_targets_diverged( &self, check: &SumTargetCheck<'_>, rows: &[serde_json::Value], ) -> bool { - if !self.calvin.ollp_is_group_leader { - return false; - } let key = ( DatabaseId::new(check.database_id), TenantId::new(check.tid), @@ -152,7 +149,7 @@ impl CoreLoop { check: &SumTargetCheck<'_>, doc_ids: &[nodedb_types::StorageKey], ) -> bool { - if !self.calvin.ollp_is_group_leader || !self.declares_materialized_sums(check) { + if !self.declares_materialized_sums(check) { return false; } let mut rows: Vec = Vec::with_capacity(doc_ids.len()); diff --git a/nodedb/src/data/executor/handlers/bulk_dml/admission.rs b/nodedb/src/data/executor/handlers/bulk_dml/admission.rs index 9e5a36920..eca591263 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/admission.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/admission.rs @@ -44,11 +44,9 @@ impl CoreLoop { /// WITHOUT writing. /// /// The apply set is the carried surrogate prediction when there is one, NOT - /// the local scan: that is the determinism anchor for multi-replica OLLP — - /// every replica, leader and follower, mutates exactly the leader's verified - /// set, so all replicas mutate identical state. With no prediction (the - /// single-shard / non-OLLP path) the local scan stands as the apply set, - /// unchanged. + /// the local scan. The prediction is checked against the scan first. With + /// no prediction (the single-shard / non-OLLP path) the local scan stands + /// as the apply set, unchanged. pub(in crate::data::executor) fn admit_bulk_predicate_write( &self, database_id: u64, @@ -59,9 +57,7 @@ impl CoreLoop { let apply_ids: Vec = match admission.predicted_surrogates { Some(predicted) => { // The set comparison is deterministic: both sides are sorted. - if self.calvin.ollp_is_group_leader - && !super::scan::ollp_surrogates_match(&matching_ids, predicted) - { + if !super::scan::ollp_surrogates_match(&matching_ids, predicted) { return Err(ErrorCode::OllpRetryRequired); } super::scan::ollp_predicted_doc_ids(predicted) @@ -77,9 +73,7 @@ impl CoreLoop { // surrogate-set check above cannot see this: the surrogate set is // unchanged. This runs BEFORE any write, so `sparse.get` still returns // pre-mutation content. - if let Some(predicted) = admission.predicted_edges - && self.calvin.ollp_is_group_leader - { + if let Some(predicted) = admission.predicted_edges { let actual = self.ollp_actual_edges(database_id, tid, admission.collection, &apply_ids); if !super::scan::ollp_edges_match(actual, predicted) { return Err(ErrorCode::OllpRetryRequired); diff --git a/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs b/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs index 967c83fc9..d117deee4 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs @@ -107,7 +107,6 @@ impl CoreLoop { epoch, position, epoch_system_ms, - is_group_leader, } = ctx; let vshard_id = task.request.vshard_id.as_u32(); debug!( @@ -116,7 +115,6 @@ impl CoreLoop { position, epoch_system_ms, vshard_id, - is_group_leader, plan_count = plans.len(), injected_count = injected_reads.len(), "calvin execute active" @@ -133,13 +131,8 @@ impl CoreLoop { // OLLP verification runs HERE, before staging, so a predicate-drift // mismatch surfaces on THIS stage response (where the scheduler releases - // locks and re-recons) and nothing is staged, resolved, or WAL-appended. - // Scoped to this replica's staged leadership for the check, then the - // resting (authoritative) state is restored. - let prev_group_leader = self.calvin.ollp_is_group_leader; - self.calvin.ollp_is_group_leader = is_group_leader; + // locks and re-recons) and nothing is staged or resolved. let verified = self.verify_calvin_active_ollp(task, tenant_id.as_u64(), plans); - self.calvin.ollp_is_group_leader = prev_group_leader; match verified { Ok(true) => {} Ok(false) => return self.calvin_stage_refusal(task, ErrorCode::OllpRetryRequired), @@ -243,7 +236,6 @@ mod tests { epoch: 1, position: 0, epoch_system_ms: 0, - is_group_leader: true, }; let injected = BTreeMap::new(); @@ -304,7 +296,6 @@ mod tests { epoch: 1, position: 0, epoch_system_ms: 0, - is_group_leader: true, }; let injected = BTreeMap::new(); @@ -340,7 +331,6 @@ mod tests { epoch: epoch_outside_synthetic_range, position: 0, epoch_system_ms: 0, - is_group_leader: true, }; let resp = core.execute_calvin_execute_active( diff --git a/nodedb/src/data/executor/handlers/control/calvin/discard.rs b/nodedb/src/data/executor/handlers/control/calvin/discard.rs index 0457561eb..f478165bd 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/discard.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/discard.rs @@ -12,9 +12,8 @@ impl CoreLoop { /// Discard a staged Calvin transaction. /// /// Removes the plans staged under `(epoch, position, vshard)` from the - /// commit-pending buffer and fires nothing — no base mutation, no side - /// effects. An - /// absent key (already flushed or dropped) is an idempotent no-op. + /// commit-pending buffer and fires nothing: no base mutation, no side + /// effects. An absent key (already installed or dropped) is a no-op. pub(in crate::data::executor) fn execute_calvin_drop( &mut self, task: &ExecutionTask, @@ -30,10 +29,6 @@ impl CoreLoop { // Discard the synthetic overlay entry alongside the raw plan buffer; // idempotent no-op if it was never staged or already removed. self.drop_calvin_synthetic_overlay(epoch, position, vshard_id); - // Writes waiting on the rows this transaction owned run next. - self.calvin - .fence - .note_resolved((epoch, position, vshard_id), None); debug!( core = self.core_id, epoch, position, vshard_id, existed, "calvin drop: discarding staged commit" @@ -64,7 +59,6 @@ mod tests { epoch: 1, position: 0, epoch_system_ms: 0, - is_group_leader: true, }; let resp = core.execute_calvin_execute_static(&task, ctx, &tenant_id, &plans, &[], &[]); assert_eq!(resp.status, Status::Ok); diff --git a/nodedb/src/data/executor/handlers/control/calvin/flush.rs b/nodedb/src/data/executor/handlers/control/calvin/flush.rs index f6a618577..de4b8fc23 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/flush.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/flush.rs @@ -117,8 +117,6 @@ impl CoreLoop { .map(|pending| pending.reply) .unwrap_or_default(); self.drop_calvin_synthetic_overlay(epoch, position, vshard_id); - // Writes waiting on the rows this transaction owned run next. - self.calvin.fence.note_resolved(key, task.wal_lsn()); let prev_epoch_ms = self.epoch_system_ms; self.epoch_system_ms = Some(epoch_system_ms); let rendered = self.calvin_reply_payload(task, tenant_id.as_u64(), reply, &ts_installs); @@ -169,7 +167,6 @@ mod tests { epoch: 1, position: 0, epoch_system_ms: 0, - is_group_leader: true, }; /// Stage `plans` at `(1, 0)`, resolve them, and return the redo bytes. @@ -400,126 +397,4 @@ mod tests { assert!(base_row(&core, "orders", 7).is_some()); assert!(!core.calvin.commit_pending.contains_key(&(1, 0, vshard_id))); } - /// An autocommit put of row `surrogate` in "orders" at `wal_lsn`. - fn autocommit_put(surrogate: u32, value: &str, wal_lsn: u64) -> ExecutionTask { - let mut task = make_task(); - task.request.plan = PhysicalPlan::Document(DocumentOp::PointPut { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "orders"), - document_id: "o1".to_string(), - value: doc_value("a", value), - surrogate: Surrogate::new(surrogate), - pk_bytes: Vec::new(), - returning: None, - rls_filters: Vec::new(), - resolved_sum_targets: Vec::new(), - }); - task.wal_lsn = Some(Lsn::new(wal_lsn)); - task - } - - /// The flush of the transaction staged at `(1, 0)`, as a queued task. - fn flush_task(redo: Vec, lsn: u64) -> ExecutionTask { - let mut task = make_task(); - task.request.plan = - PhysicalPlan::Meta(nodedb_physical::physical_plan::MetaOp::CalvinFlush { - epoch: 1, - position: 0, - redo, - collections: vec!["orders".to_string()], - sum_targets: Vec::new(), - }); - task.wal_lsn = Some(Lsn::new(lsn)); - task - } - - fn holds(core: &CoreLoop, needle: &[u8]) -> bool { - base_row(core, "orders", 7) - .is_some_and(|row| row.windows(needle.len()).any(|w| w == needle)) - } - - /// An autocommit write to a row a staged Calvin transaction owns waits - /// for the flush, then applies on top of it. - #[test] - fn a_write_to_an_owned_row_applies_after_the_flush() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - let redo = stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - - core.task_queue.push(autocommit_put(7, "autocommit", 60)); - core.tick(); - assert_eq!(core.calvin.fence.len(), 1, "the write waits for the owner"); - assert!(base_row(&core, "orders", 7).is_none()); - - core.task_queue.push(flush_task(redo, 50)); - core.tick(); - - assert_eq!(core.calvin.fence.len(), 0); - assert!( - holds(&core, b"autocommit"), - "the waiting write applies after the flush and is not lost" - ); - } - - /// A waiting write whose record sits below the flush's redo record is - /// refused: restart replay would apply it before the flush. - #[test] - fn a_waiting_write_below_the_flush_lsn_is_refused() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - let redo = stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - - core.task_queue.push(autocommit_put(7, "autocommit", 40)); - core.task_queue.push(flush_task(redo, 50)); - core.tick(); - - assert_eq!(core.calvin.fence.len(), 0); - assert!(base_row(&core, "orders", 7).is_some()); - assert!( - !holds(&core, b"autocommit"), - "the refused write never applies" - ); - } - - /// A constraint-set install on a collection a staged transaction writes - /// runs at once. It writes no row the flush installs, and a parked - /// install holds its data group's apply loop behind the flush. - #[test] - fn a_constraint_install_on_an_owned_collection_does_not_wait() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - - let mut install = make_task(); - install.request.plan = - PhysicalPlan::Crdt(nodedb_physical::physical_plan::CrdtOp::SetConstraints { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "orders"), - constraint_version: 1, - constraints: Vec::new(), - }); - install.request.admission = crate::bridge::envelope::Admission::Exempt( - crate::bridge::envelope::ExemptReason::AlreadyOrdered, - ); - core.task_queue.push(install); - core.tick(); - - assert_eq!( - core.calvin.fence.len(), - 0, - "a constraint install never waits for a Calvin flush" - ); - } - - /// A write to a row the staged transaction does not own runs at once. - #[test] - fn a_write_to_an_unowned_row_does_not_wait() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - - core.task_queue.push(autocommit_put(8, "autocommit", 60)); - core.tick(); - - assert_eq!(core.calvin.fence.len(), 0); - assert!(base_row(&core, "orders", 8).is_some()); - } } diff --git a/nodedb/src/data/executor/handlers/control/calvin/mod.rs b/nodedb/src/data/executor/handlers/control/calvin/mod.rs index 33fb2b19c..42583c3cf 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/mod.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/mod.rs @@ -8,9 +8,9 @@ //! (the common case). It VALIDATES the read-set to compute the local commit //! vote and STAGES the transaction's plans into the synthetic overlay and //! the commit-pending buffer WITHOUT mutating base or firing side effects, -//! then returns the vote. `CoreLoop::execute_calvin_flush` later installs -//! the transaction's committed redo record, or -//! `CoreLoop::execute_calvin_drop` discards the staged state. +//! then returns the vote. The data group's apply of the slice's stamped +//! redo entry later installs it and consumes the staged state, or +//! `CoreLoop::execute_calvin_drop` discards it. //! //! - `CoreLoop::execute_calvin_execute_passive`: passive participant for a //! dependent-read txn. Reads each declared key from the local engine and @@ -26,9 +26,8 @@ //! returns `OllpRetryRequired` and stages nothing. The OLLP orchestrator on //! the Control Plane retries via `Inbox::submit`. //! -//! The scheduler appends the transaction's `TransactionRedo` record, or a -//! `CalvinApplied` marker when the transaction wrote nothing here, on the -//! Control Plane. The Data Plane writes no WAL record. +//! The data-group leader's scheduler proposes the slice's stamped redo entry +//! to the data group. The Data Plane writes no WAL record for a slice. mod active_passive; mod discard; diff --git a/nodedb/src/data/executor/handlers/control/calvin/shared.rs b/nodedb/src/data/executor/handlers/control/calvin/shared.rs index d690594b4..dc363a522 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/shared.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/shared.rs @@ -16,7 +16,6 @@ pub(in crate::data::executor) struct CalvinExecCtx { pub epoch: u64, pub position: u32, pub epoch_system_ms: i64, - pub is_group_leader: bool, } impl CoreLoop { @@ -48,9 +47,6 @@ impl CoreLoop { .commit_pending .remove(&(epoch, position, vshard_id)); self.drop_calvin_synthetic_overlay(epoch, position, vshard_id); - self.calvin - .fence - .note_resolved((epoch, position, vshard_id), None); // The scheduler proposes this abort vote and waits for the global // verdict before it drops anything. self.calvin_stage_refusal(task, error.into()) diff --git a/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs b/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs index 6e9d44ff2..2fa030878 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs @@ -29,10 +29,9 @@ impl CoreLoop { /// against the per-core write versions, then STAGES the write plans into /// the synthetic overlay and the commit-pending buffer keyed by /// `(epoch, position)`. It performs NO base mutation and fires NO side - /// effects — nothing is observable until a subsequent - /// [`CoreLoop::execute_calvin_flush`] installs the transaction's redo - /// record (or [`CoreLoop::execute_calvin_drop`] discards the staged - /// state). The response carries the vote on `stage_vote`. Staging runs + /// effects. Nothing is observable until the apply of the slice's stamped + /// redo entry installs it, or [`CoreLoop::execute_calvin_drop`] discards + /// the staged state. The response carries the vote on `stage_vote`. Staging runs /// under the epoch's deterministic time anchor, which the pending entry /// keeps for resolve. `body_plans` indexes the plans a trigger body /// buffered: they stage under `Trigger`, so their rows fire no trigger. @@ -49,7 +48,6 @@ impl CoreLoop { epoch, position, epoch_system_ms, - is_group_leader, } = ctx; let vshard_id = task.request.vshard_id.as_u32(); debug!( @@ -58,7 +56,6 @@ impl CoreLoop { position, epoch_system_ms, vshard_id, - is_group_leader, plan_count = plans.len(), read_count = versioned_reads.len(), "calvin stage for commit" @@ -215,7 +212,6 @@ mod tests { epoch: 1, position: 0, epoch_system_ms: 0, - is_group_leader: true, }; let resp = core.execute_calvin_execute_static(&task, ctx, &tenant_id, &plans, &[], &[]); @@ -274,7 +270,6 @@ mod tests { epoch: 3, position: 0, epoch_system_ms: 0, - is_group_leader: true, }; let resp = core.execute_calvin_execute_static( @@ -310,7 +305,6 @@ mod tests { epoch: 3, position: 1, epoch_system_ms: 0, - is_group_leader: true, }; let resp = core.execute_calvin_execute_static( @@ -340,7 +334,6 @@ mod tests { epoch: epoch_outside_synthetic_range, position: 0, epoch_system_ms: 0, - is_group_leader: true, }, &tenant_id, &[point_insert_plan("orders", "o1", 7)], @@ -373,7 +366,6 @@ mod tests { epoch: 9, position: 3, epoch_system_ms: 0, - is_group_leader: true, }; let vshard = task.request.vshard_id.as_u32(); let synthetic = calvin_synthetic_txn_id(9, 3, vshard).unwrap(); @@ -422,7 +414,6 @@ mod tests { epoch: 9, position: 5, epoch_system_ms: 0, - is_group_leader: true, }, &tenant_id, &[bulk_delete_plan("orders", None)], @@ -453,7 +444,6 @@ mod tests { epoch: 9, position: 4, epoch_system_ms: 0, - is_group_leader: true, }; let vshard = task.request.vshard_id.as_u32(); let synthetic = calvin_synthetic_txn_id(9, 4, vshard).unwrap(); @@ -515,7 +505,6 @@ mod tests { epoch: 21, position: 1, epoch_system_ms: 0, - is_group_leader: true, }, &tenant, &[malformed], @@ -537,7 +526,6 @@ mod tests { epoch: 21, position: 1, epoch_system_ms: 0, - is_group_leader: true, }, &tenant, &[valid], @@ -559,7 +547,6 @@ mod tests { epoch: 21, position: 2, epoch_system_ms: 0, - is_group_leader: true, }, &tenant, &[mismatch], @@ -583,7 +570,6 @@ mod tests { epoch: 21, position: 3, epoch_system_ms: 0, - is_group_leader: true, }, &tenant, &[overflow], diff --git a/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs b/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs index 9126c4b21..bef35504f 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs @@ -47,7 +47,6 @@ impl CoreLoop { epoch, position: 0, epoch_system_ms, - is_group_leader: true, }; let staged = self.execute_calvin_execute_static(task, ctx, &TenantId::new(tid), plans, &[], &[]); diff --git a/nodedb/src/data/executor/handlers/control/calvin_active_verify.rs b/nodedb/src/data/executor/handlers/control/calvin_active_verify.rs index 189c9e2af..fa82109ba 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_active_verify.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_active_verify.rs @@ -16,8 +16,8 @@ use crate::data::executor::handlers::bulk_dml::scan::{ use crate::data::executor::task::ExecutionTask; impl CoreLoop { - /// Leader-only OLLP verification of every predicate-DML plan in a - /// dependent-read ACTIVE Calvin txn, before staging. `Ok(false)` means a + /// OLLP verification of every predicate-DML plan in a dependent-read + /// ACTIVE Calvin txn, before staging. Only the data-group leader stages. `Ok(false)` means a /// prediction drifted; caller returns `OllpRetryRequired`, stages nothing. pub(in crate::data::executor) fn verify_calvin_active_ollp( &self, @@ -25,9 +25,6 @@ impl CoreLoop { tid: u64, plans: &[PhysicalPlan], ) -> crate::Result { - if !self.calvin.ollp_is_group_leader { - return Ok(true); - } let database_id = task.request.database_id.as_u64(); for plan in plans { // Only the document engine carries an OLLP prediction; exhaustive diff --git a/nodedb/src/data/executor/handlers/control/calvin_resolve.rs b/nodedb/src/data/executor/handlers/control/calvin_resolve.rs index 452ee9ef7..c06ea8b37 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_resolve.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_resolve.rs @@ -143,7 +143,7 @@ mod tests { /// A minimal `ExecutionTask` homing to vShard 0, tenant 1, database /// DEFAULT, matching what the Calvin scheduler dispatches with (see /// `dispatch.rs`'s `DatabaseId::DEFAULT` for `CalvinExecuteStatic` / - /// `CalvinFlush` / `CalvinDrop`; `CalvinResolve` must match). + /// `CalvinDrop`; `CalvinResolve` must match). fn make_task() -> ExecutionTask { let plan = PhysicalPlan::Document(DocumentOp::PointGet { collection: QualifiedCollection::new(DatabaseId::DEFAULT, "x"), @@ -297,7 +297,6 @@ mod tests { epoch, position, epoch_system_ms: 0, - is_group_leader: true, }; let resp = core.execute_calvin_execute_static(task, ctx, &TenantId::new(1), plans, &[], &[]); @@ -489,7 +488,6 @@ mod tests { epoch: 6, position: 0, epoch_system_ms: 0, - is_group_leader: true, }; let resp = core.execute_calvin_execute_static( &task, @@ -631,7 +629,6 @@ mod tests { epoch: 8, position: 3, epoch_system_ms: EPOCH_MS, - is_group_leader: true, }; let staged = core.execute_calvin_execute_static( &task, diff --git a/nodedb/src/data/executor/handlers/control/mod.rs b/nodedb/src/data/executor/handlers/control/mod.rs index a645588d5..5a2284b64 100644 --- a/nodedb/src/data/executor/handlers/control/mod.rs +++ b/nodedb/src/data/executor/handlers/control/mod.rs @@ -26,4 +26,5 @@ pub mod reindex; pub mod snapshot; pub mod synonym_group; +#[cfg(test)] pub(in crate::data::executor) use calvin_txn_id::calvin_synthetic_txn_id; diff --git a/nodedb/src/data/executor/handlers/control/reindex/waiter.rs b/nodedb/src/data/executor/handlers/control/reindex/waiter.rs index 0e43d6aae..83006404a 100644 --- a/nodedb/src/data/executor/handlers/control/reindex/waiter.rs +++ b/nodedb/src/data/executor/handlers/control/reindex/waiter.rs @@ -4,10 +4,9 @@ //! //! The rebuilds run exactly as for REINDEX CONCURRENTLY: off the core, with //! the core serving every other request meanwhile. Only the answer waits. -//! The core holds the request's task here and answers it from the tick, -//! the way it holds a write parked behind a staged Calvin transaction -//! (`core_loop::calvin_fence`): the held task builds the response, the -//! response ring carries it, and the task's deadline bounds the wait. +//! The core holds the request's task here and answers it from the tick: +//! the held task builds the response, the response ring carries it, and the +//! task's deadline bounds the wait. //! //! - Every rebuild cut over: `Ok`. //! - A rebuild was discarded, or an HNSW segment rebuild failed: the first diff --git a/nodedb/src/data/executor/handlers/point/apply_put/types.rs b/nodedb/src/data/executor/handlers/point/apply_put/types.rs index f9082bb4d..71ee70832 100644 --- a/nodedb/src/data/executor/handlers/point/apply_put/types.rs +++ b/nodedb/src/data/executor/handlers/point/apply_put/types.rs @@ -149,6 +149,7 @@ pub(in crate::data::executor) fn map_enforcement_error(e: ErrorCode) -> crate::E | ErrorCode::RejectedConstraint { .. } | ErrorCode::RejectedPrevalidation { .. } | ErrorCode::RetryableRefusal { .. } + | ErrorCode::CoreFailStopped { .. } | ErrorCode::SyncRejected { .. } | ErrorCode::SyncNotApplied { .. } | ErrorCode::NotFound diff --git a/nodedb/src/data/executor/handlers/snapshot/restore/tenant_snapshot.rs b/nodedb/src/data/executor/handlers/snapshot/restore/tenant_snapshot.rs index 60306fbdb..de2cc38cb 100644 --- a/nodedb/src/data/executor/handlers/snapshot/restore/tenant_snapshot.rs +++ b/nodedb/src/data/executor/handlers/snapshot/restore/tenant_snapshot.rs @@ -496,6 +496,7 @@ mod tests { PhysicalPlan::Meta(MetaOp::WalAppend { payload: Vec::new(), }), + None, ); let response = core.execute_restore_tenant_snapshot( @@ -568,6 +569,7 @@ mod tests { PhysicalPlan::Meta(MetaOp::WalAppend { payload: Vec::new(), }), + None, ); (bytes, task) } @@ -653,6 +655,7 @@ mod tests { PhysicalPlan::Meta(MetaOp::WalAppend { payload: Vec::new(), }), + None, ); let response = core.execute_restore_tenant_snapshot(&task, 0, &bytes, true, &[], &[]); assert_eq!(response.status, Status::Ok, "restore must succeed"); diff --git a/nodedb/src/data/executor/handlers/transaction/mod.rs b/nodedb/src/data/executor/handlers/transaction/mod.rs index 5d6e8c40d..b99ec2f07 100644 --- a/nodedb/src/data/executor/handlers/transaction/mod.rs +++ b/nodedb/src/data/executor/handlers/transaction/mod.rs @@ -8,5 +8,9 @@ pub(in crate::data::executor) mod resolve; pub(in crate::data::executor) mod savepoint_marks; pub(in crate::data::executor) mod stage_write; pub(in crate::data::executor) mod undo; +#[cfg(test)] mod write_version; +#[cfg(test)] mod write_version_kv; +#[cfg(test)] +mod write_version_parity_tests; diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_fold_tests.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_fold_tests.rs index 84bb5591f..476d72b94 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_fold_tests.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_fold_tests.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: BUSL-1.1 -//! A Calvin redo record folds its materialized sums from its stamp, the same -//! way live and in restart replay. +//! A Calvin redo record folds its materialized sums at its live install. The +//! fold rows travel in the install's write set, so the funnel journals them +//! as parts of the record's group, and restart replay applies those parts. use nodedb_physical::physical_plan::{MaterializedSumBinding, RedoSumTargets, ResolvedSumTarget}; use nodedb_types::{DatabaseId, Surrogate, TenantId}; @@ -11,12 +12,13 @@ use nodedb_wal::{TombstoneSet, WalRecord}; use super::entry::CommittedRedo; use super::test_commit::doc_put_sub_record; use crate::bridge::envelope::Status; +use crate::control::server::wal_dispatch::{GroupOrigin, WriteSetTarget, plan_group_parts}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; use crate::data::executor::doc_format; use crate::engine::document::store::CollectionConfig; -use crate::types::Lsn; -use crate::wal::{CalvinStamp, RedoRecord}; +use crate::types::{Lsn, VShardId}; +use crate::wal::{CalvinStamp, RedoRecord, WriteGroup, WriteGroupRecord}; const DB: u64 = 0; pub(super) const TID: u64 = 1; @@ -82,8 +84,6 @@ pub(super) fn calvin_redo() -> Vec { epoch: 3, position: 0, vshard_id: 0, - collections: vec![SOURCE.to_string()], - sum_targets: sum_targets(), }), cross_shard_applied: None, row_sources: Vec::new(), @@ -111,8 +111,27 @@ pub(super) fn balance(core: &CoreLoop) -> Option { .map(str::to_string) } +/// The WAL record at `lsn` of `record_type` carrying `payload`. +fn wal_record(record_type: RecordType, lsn: u64, payload: Vec) -> WalRecord { + WalRecord::new(WalRecordArgs { + record_type: record_type as u32, + lsn, + tenant_id: TID, + vshard_id: 0, + database_id: DB, + payload, + encryption_key: None, + preamble_bytes: None, + }) + .expect("wal record") +} + +/// The live install folds the target row and reports it in its write set. +/// Restart replay of the record and the parts journalled from that write +/// set reaches the live total. A replay of the record alone folds nothing: +/// the stamp carries no fold. #[test] -fn restart_replay_folds_a_calvin_record_to_the_live_total() { +fn restart_replay_applies_the_journalled_fold_rows_not_the_stamp() { let redo = calvin_redo(); let live_dir = tempfile::tempdir().expect("tempdir"); @@ -130,35 +149,60 @@ fn restart_replay_folds_a_calvin_record_to_the_live_total() { ); assert_eq!(response.status, Status::Ok, "{:?}", response.error_code); assert_eq!(balance(&live).as_deref(), Some("125")); + assert!( + !response.write_set.is_empty(), + "the fold target row travels in the write set" + ); + + let parts = plan_group_parts( + &WriteSetTarget { + tenant_id: TenantId::new(TID), + vshard_id: VShardId::new(0), + database_id: DatabaseId::DEFAULT, + collection: SOURCE, + origin: GroupOrigin { lsn: Lsn::new(LSN) }, + }, + &response.write_set, + 1 << 20, + ) + .expect("plan parts"); + let count = parts.count().expect("part count"); + let mut records = vec![wal_record(RecordType::TransactionRedo, LSN, redo.clone())]; + for (part, (_, ops)) in (1..=count).zip(parts.runs) { + let group = WriteGroupRecord { + group: WriteGroup::part_of(LSN, part, count), + ops, + redo: None, + }; + records.push(wal_record( + RecordType::WriteGroup, + LSN + u64::from(part), + group.to_bytes().expect("encode part"), + )); + } let replay_dir = tempfile::tempdir().expect("tempdir"); let (mut replayed, _replay_ends) = core_with_sum(replay_dir.path()); - let record = WalRecord::new(WalRecordArgs { - record_type: RecordType::TransactionRedo as u32, - lsn: LSN, - tenant_id: TID, - vshard_id: 0, - database_id: DB, - payload: redo, - encryption_key: None, - preamble_bytes: None, - }) - .expect("wal record"); replayed - .replay_transaction_redo_wal(std::slice::from_ref(&record), 1, &TombstoneSet::new()) + .replay_transaction_redo_wal(&records, 1, &TombstoneSet::new()) .expect("replay"); assert_eq!( balance(&replayed), balance(&live), - "restart replay folds to the total the live install produced" + "restart replay of the record and its parts reaches the live total" ); - replayed - .replay_transaction_redo_wal(std::slice::from_ref(&record), 1, &TombstoneSet::new()) - .expect("replay again"); + let bare_dir = tempfile::tempdir().expect("tempdir"); + let (mut bare, _bare_ends) = core_with_sum(bare_dir.path()); + bare.replay_transaction_redo_wal( + &[wal_record(RecordType::TransactionRedo, LSN, redo)], + 1, + &TombstoneSet::new(), + ) + .expect("replay record alone"); assert_eq!( - balance(&replayed).as_deref(), - Some("125"), - "a second replay over the installed row folds nothing" + balance(&bare).as_deref(), + Some("100"), + "the record alone folds nothing" ); } diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install.rs index fa37fa829..3139d745d 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install.rs @@ -47,6 +47,8 @@ impl CoreLoop { position, epoch_system_ms, reply, + // The Control Plane raises the write mark; the install writes rows. + user_write: _, } = install; let (epoch, position, epoch_system_ms) = (*epoch, *position, *epoch_system_ms); let vshard_id = task.request.vshard_id.as_u32(); @@ -73,11 +75,10 @@ impl CoreLoop { return response; } // The record installed: a staged entry of the slice and its overlay - // are spent. Writes waiting on the rows the entry owned run next. - let key = (epoch, position, vshard_id); - if self.calvin.commit_pending.remove(&key).is_some() { - self.calvin.fence.note_resolved(key, task.wal_lsn()); - } + // are spent. + self.calvin + .commit_pending + .remove(&(epoch, position, vshard_id)); self.drop_calvin_synthetic_overlay(epoch, position, vshard_id); let prev_epoch_ms = self.epoch_system_ms; diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs index fe5e55de2..a1b990efe 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs @@ -55,7 +55,6 @@ fn stage_and_resolve(core: &mut CoreLoop, plans: &[PhysicalPlan]) -> CalvinResol epoch: 1, position: 0, epoch_system_ms: 0, - is_group_leader: true, }; let staged = core.execute_calvin_execute_static(&task, ctx, &TenantId::new(TID), plans, &[], &[]); @@ -83,6 +82,7 @@ fn install_plan( position: 0, epoch_system_ms: 0, reply, + user_write: true, }), }) } diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/document.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/document.rs index 6ae8bb6f0..71b99df6f 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/document.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/document.rs @@ -92,7 +92,7 @@ impl CoreLoop { row: &CommittedDocWrite<'_>, value: &[u8], ) -> crate::Result<()> { - let (resolved, deferred) = self.committed_sum_targets(row.collection, row.record_lsn); + let (resolved, deferred) = self.committed_sum_targets(row.collection); let surrogate = Surrogate::new(row.surrogate); let storage_key = StorageKey::for_surrogate(surrogate); let wal_lsn = (row.record_lsn != 0).then(|| Lsn::new(row.record_lsn)); @@ -238,7 +238,7 @@ impl CoreLoop { } fn committed_document_delete(&mut self, row: &CommittedDocWrite<'_>) -> crate::Result { - let (resolved, _) = self.committed_sum_targets(row.collection, row.record_lsn); + let (resolved, _) = self.committed_sum_targets(row.collection); let surrogate = Surrogate::new(row.surrogate); let storage_key = StorageKey::for_surrogate(surrogate); let hook_ctx = HookCtx { @@ -333,24 +333,20 @@ impl CoreLoop { Ok(true) } - /// The sum targets a write to `collection` folds into: the open scope's - /// under a committed-redo apply, else the restart-replay folds of the - /// record at `record_lsn`. + /// The sum targets a write to `collection` folds into: the open + /// committed-redo scope's. With no scope open nothing folds. fn committed_sum_targets( &self, collection: &str, - record_lsn: u64, ) -> ( Vec, Vec, ) { - match self.redo_apply.scope.as_ref() { - Some(scope) => scope.sum_targets_for(collection), - None => self - .redo_apply - .replay_folds_for(record_lsn, collection) - .unwrap_or_default(), - } + self.redo_apply + .scope + .as_ref() + .map(|scope| scope.sum_targets_for(collection)) + .unwrap_or_default() } fn record_committed_doc_write(&mut self, write: AppliedDocWrite) { diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/mod.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/mod.rs index 0bb273351..042b0b49f 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/mod.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/mod.rs @@ -15,8 +15,8 @@ //! - [`test_commit`]: a test driver for a session commit on one core. //! - [`install_refusal_tests`]: one committed and one refused install per //! engine kind. -//! - [`calvin_fold_tests`]: a Calvin record folds the same live and in -//! restart replay. +//! - [`calvin_fold_tests`]: a Calvin record's fold rows journal as parts, +//! and restart replay applies them. //! - [`calvin_install_tests`]: a Calvin slice installs from its stamped //! redo entry. //! - [`unique_handover_tests`]: UNIQUE judged on a record's post-state. diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs index ef0395978..a6f7fb22c 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs @@ -32,54 +32,16 @@ pub(in crate::data::executor) struct RedoApplyState { pub(in crate::data::executor) num_cores: usize, /// `Some` only while one committed redo record applies on this core. pub(in crate::data::executor) scope: Option, - /// During restart replay: the materialized-sum targets each Calvin redo - /// record's stamp carries, keyed by the record's LSN. The document redo - /// arm folds a row at such an LSN into its targets, as the live install - /// did. Empty outside restart replay. - pub(in crate::data::executor) replay_folds: HashMap>, } impl RedoApplyState { - /// The sum targets restart replay folds a write to `collection` at - /// `record_lsn` into, when the record carries any. - pub(in crate::data::executor) fn replay_folds_for( - &self, - record_lsn: u64, - collection: &str, - ) -> Option<(Vec, Vec)> { - self.replay_folds - .get(&record_lsn)? - .iter() - .find(|targets| targets.collection == collection) - .map(|targets| (targets.resolved.clone(), targets.deferred.clone())) - } - /// A single-core default. Every multi-core runtime sets the real count /// through `CoreLoop::set_num_cores` before the core serves requests. pub(in crate::data::executor) fn new() -> Self { Self { num_cores: 1, scope: None, - replay_folds: HashMap::new(), - } - } -} - -impl crate::data::executor::core_loop::CoreLoop { - /// Arm restart replay with the sum targets each Calvin record carries. - /// Returns whether this is restart replay: a committed-redo apply folds - /// from its open scope instead, and leaves them unused. - pub(crate) fn begin_replay_folds(&mut self, folds: HashMap>) -> bool { - let restart = self.redo_apply.scope.is_none(); - if restart { - self.redo_apply.replay_folds = folds; } - restart - } - - /// Drop the restart-replay sum targets once the document redo arm ran. - pub(crate) fn end_replay_folds(&mut self) { - self.redo_apply.replay_folds.clear(); } } diff --git a/nodedb/src/data/executor/handlers/transaction/write_version.rs b/nodedb/src/data/executor/handlers/transaction/write_version.rs index 34f6bc683..2d76e597b 100644 --- a/nodedb/src/data/executor/handlers/transaction/write_version.rs +++ b/nodedb/src/data/executor/handlers/transaction/write_version.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Records the per-core last-write-LSN version for every key a committed -//! transaction batch wrote. Runs once, after commit, over the buffered -//! sub-plans, covering both the fast-path commit and every Calvin apply. +//! The per-key write versions a committed batch of plans implies, read off +//! the plans. The redo install records versions through its replay arms. +//! The parity tests check the install against this oracle, engine by engine. //! One WAL LSN applies to every key in the batch. use crate::bridge::envelope::PhysicalPlan; @@ -19,7 +19,7 @@ impl CoreLoop { /// Record the version of every key written by a committed transaction /// batch. No-op with no WAL LSN. Per-key engines record `KeyRepr`; /// engines with internal per-key identity record only the collection floor. - pub(in crate::data::executor) fn record_batch_write_versions( + pub(super) fn record_batch_write_versions( &mut self, task: &ExecutionTask, tid: u64, diff --git a/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs b/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs index 5e8712292..fc8cfc11d 100644 --- a/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs +++ b/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: BUSL-1.1 -//! `KvOp` write-version recording, split out of `write_version.rs` to keep -//! that file under the size limit. Same contract: no-op with no WAL LSN, -//! one `note_write_lsn` per key a committed KV write touched. +//! `KvOp` write versions for the parity oracle in `write_version.rs`. Same +//! contract: no-op with no WAL LSN, one `note_write_lsn` per key a committed +//! KV write touched. use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::write_index::KeyRepr; diff --git a/nodedb/src/data/executor/handlers/transaction/write_version_parity_tests.rs b/nodedb/src/data/executor/handlers/transaction/write_version_parity_tests.rs new file mode 100644 index 000000000..31196cb32 --- /dev/null +++ b/nodedb/src/data/executor/handlers/transaction/write_version_parity_tests.rs @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A committed Calvin slice installs through its stamped redo entry, and the +//! install records every write version the slice's plans imply. +//! +//! The oracle is `record_batch_write_versions` over the slice's plans. For +//! each engine, every per-key version and collection floor the oracle records +//! holds on the installed core at the install's LSN or above. + +use nodedb_physical::physical_plan::{ + CalvinInstall, CalvinReplySpec, CalvinResolved, ColumnarInsertIntent, ColumnarOp, CrdtOp, + CrdtWriteVerb, DocumentOp, GraphOp, KvOp, MetaOp, PhysicalPlan, RedoOrigin, SpatialOp, TextOp, + TimeseriesOp, VectorOp, +}; +use nodedb_types::geometry::Geometry; +use nodedb_types::{DatabaseId, QualifiedCollection, RlsWriteCheck, Surrogate, Value}; + +use crate::bridge::envelope::{Response, Status}; +use crate::control::wal_replication::transaction_redo::collections::written_collections; +use crate::control::wal_replication::transaction_redo::sum_targets::redo_sum_targets; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; +use crate::data::executor::handlers::control::calvin::CalvinExecCtx; +use crate::data::executor::task::ExecutionTask; +use crate::types::Lsn; +use crate::wal::{RedoRecord, RedoSubRecord}; + +/// The LSN every install and every oracle record names. +const LSN: u64 = 300; + +/// A core with its bridge ends and data directory kept alive. +struct Core { + core: CoreLoop, + _req: Box, + _resp: Box, + _dir: tempfile::TempDir, +} + +fn fresh_core() -> Core { + let dir = tempfile::tempdir().expect("tempdir"); + let (core, req, resp) = make_core_with_dir(dir.path()); + Core { + core, + _req: Box::new(req), + _resp: Box::new(resp), + _dir: dir, + } +} + +fn tid() -> u64 { + make_default_task().request.tenant_id.as_u64() +} + +fn task_at(lsn: u64) -> ExecutionTask { + let mut task = make_default_task(); + task.wal_lsn = Some(Lsn::new(lsn)); + task +} + +fn object(fields: &[(&str, Value)]) -> Vec { + let map: std::collections::HashMap = fields + .iter() + .map(|(k, v)| ((*k).to_string(), v.clone())) + .collect(); + nodedb_types::value_to_msgpack(&Value::Object(map)).expect("encode object") +} + +/// Stage `plans` as the slice at `(1, 0)` and resolve its redo. +fn stage_and_resolve(core: &mut CoreLoop, plans: &[PhysicalPlan]) -> CalvinResolved { + let task = make_default_task(); + let ctx = CalvinExecCtx { + epoch: 1, + position: 0, + epoch_system_ms: 0, + }; + let tenant = task.request.tenant_id; + let staged = core.execute_calvin_execute_static(&task, ctx, &tenant, plans, &[], &[]); + assert_eq!(staged.status, Status::Ok, "{:?}", staged.error_code); + let resolved = core.execute_calvin_resolve(&task, 1, 0); + assert_eq!(resolved.status, Status::Ok, "{:?}", resolved.error_code); + zerompk::from_msgpack(resolved.payload.as_bytes()).expect("decode resolved answer") +} + +/// Install `redo` as the stamped slice at `(1, 0)` at [`LSN`]. +fn install( + core: &mut CoreLoop, + redo: Vec, + plans: &[PhysicalPlan], + reply: CalvinReplySpec, +) -> Response { + let plan = PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { + redo, + collections: written_collections(plans), + sum_targets: redo_sum_targets(plans), + origin: RedoOrigin::Commit, + calvin: Some(CalvinInstall { + epoch: 1, + position: 0, + epoch_system_ms: 0, + reply, + user_write: true, + }), + }); + let installed = core.execute_plan(&task_at(LSN), &plan); + assert_eq!(installed.status, Status::Ok, "{:?}", installed.error_code); + installed +} + +/// Check that `installed` holds every version the oracle records for +/// `plans`. The oracle must record at least one collection floor. +fn assert_parity(installed: &CoreLoop, plans: &[PhysicalPlan]) { + let mut oracle = fresh_core(); + oracle + .core + .record_batch_write_versions(&task_at(LSN), tid(), plans); + let floors: Vec<_> = oracle.core.write_index.recorded_collections().collect(); + assert!(!floors.is_empty(), "the oracle records the slice's writes"); + for (key, lsn) in floors { + let held = installed.write_index.collection_write_lsn(key); + assert!( + held.is_some_and(|held| held >= lsn), + "the install records the collection floor of {key:?}: {held:?}" + ); + } + for (key, lsn) in oracle.core.write_index.recorded_keys() { + let held = installed.write_index.key_write_lsn(key); + assert!( + held.is_some_and(|held| held >= lsn), + "the install records the version of {key:?}: {held:?}" + ); + } +} + +/// Stage, resolve and install `plans` as one Calvin slice on a fresh core, +/// then check parity. +fn staged_slice_parity(plans: &[PhysicalPlan]) { + let mut core = fresh_core(); + let resolved = stage_and_resolve(&mut core.core, plans); + install(&mut core.core, resolved.redo, plans, resolved.reply); + assert_parity(&core.core, plans); +} + +#[test] +fn a_document_slice_records_every_row_version() { + let plans = [ + PhysicalPlan::Document(DocumentOp::PointInsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "notes"), + document_id: "n1".to_string(), + value: object(&[("title", Value::String("kept".into()))]), + if_absent: false, + surrogate: Surrogate::new(41), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: Vec::new(), + }), + PhysicalPlan::Document(DocumentOp::PointPut { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "notes"), + document_id: "n2".to_string(), + value: object(&[("title", Value::String("also".into()))]), + surrogate: Surrogate::new(42), + pk_bytes: b"n2".to_vec(), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + }), + ]; + staged_slice_parity(&plans); +} + +#[test] +fn a_kv_slice_records_every_key_version() { + let put = |key: &[u8], surrogate: u32| { + PhysicalPlan::Kv(KvOp::Put { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "cache"), + key: key.to_vec(), + value: object(&[("n", Value::Integer(1))]), + ttl_ms: 0, + surrogate: Surrogate::new(surrogate), + returning: None, + rls_filters: Vec::new(), + provenance: None, + }) + }; + staged_slice_parity(&[put(b"k1", 51), put(b"k2", 52)]); +} + +#[test] +fn a_vector_slice_records_every_surrogate_version() { + let plan = PhysicalPlan::Vector(VectorOp::DirectInsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "vp"), + field: "vec".into(), + surrogate: Surrogate::new(61), + pk_bytes: b"r".to_vec(), + vector: vec![1.0, 0.0], + payload: zerompk::to_msgpack_vec(&std::collections::HashMap::from([( + "id".to_string(), + Value::String("r".into()), + )])) + .expect("encode payload"), + quantization: nodedb_types::VectorQuantization::None, + storage_dtype: nodedb_types::VectorStorageDtype::F32, + payload_indexes: Vec::new(), + returning: None, + rls_filters: Vec::new(), + }); + staged_slice_parity(&[plan]); +} + +#[test] +fn a_graph_slice_records_every_edge_version() { + let plan = PhysicalPlan::Graph(GraphOp::EdgePut { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "social"), + src_id: "a".into(), + label: "KNOWS".into(), + dst_id: "b".into(), + properties: Vec::new(), + src_surrogate: Surrogate::new(1), + dst_surrogate: Surrogate::new(2), + }); + staged_slice_parity(&[plan]); +} + +#[test] +fn a_columnar_slice_records_its_collection_floor() { + let rows = Value::Array(vec![Value::Object(std::collections::HashMap::from([ + ("id".to_string(), Value::Integer(1)), + ("v".to_string(), Value::Integer(7)), + ]))]); + let plan = PhysicalPlan::Columnar(ColumnarOp::Insert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "metrics_c"), + payload: nodedb_types::value_to_msgpack(&rows).expect("encode rows"), + format: "msgpack".into(), + intent: ColumnarInsertIntent::Insert, + on_conflict_updates: Vec::new(), + surrogates: vec![Surrogate::new(71)], + schema_bytes: Vec::new(), + provenance: None, + wal_lsn: None, + rls_write_check: RlsWriteCheck::already_decided_elsewhere(), + returning: None, + rls_filters: Vec::new(), + }); + staged_slice_parity(&[plan]); +} + +#[test] +fn a_timeseries_slice_records_its_collection_floor() { + let plan = PhysicalPlan::Timeseries(TimeseriesOp::Ingest { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "metrics"), + payload: b"metrics,host=a value=1 1000000000".to_vec(), + format: "ilp".to_owned(), + wal_lsn: None, + surrogates: Vec::new(), + provenance: None, + rls_write_check: RlsWriteCheck::NoPolicyApplies, + returning: None, + rls_filters: Vec::new(), + }); + staged_slice_parity(&[plan]); +} + +#[test] +fn a_spatial_slice_records_its_collection_floor() { + let plan = PhysicalPlan::Spatial(SpatialOp::Insert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "places"), + field: "geom".to_string(), + surrogate: Surrogate::new(81), + geometry: Geometry::Point { + coordinates: [1.0, 2.0], + }, + provenance: None, + }); + staged_slice_parity(&[plan]); +} + +#[test] +fn a_crdt_slice_records_its_collection_floor() { + let plan = PhysicalPlan::Crdt(CrdtOp::DocUpsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "tasks"), + document_id: "t1".to_string(), + fields_json: r#"{"title":"kept"}"#.to_string(), + surrogate: Surrogate::new(71), + partial: false, + verb: CrdtWriteVerb::Insert, + returning: None, + rls_filters: Vec::new(), + }); + staged_slice_parity(&[plan]); +} + +/// A text write stages no row. Its slice installs the sub-record its +/// resolve serializes from the plan. +#[test] +fn a_text_slice_records_its_collection_floor() { + let op = TextOp::FtsIndexDoc { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "docs"), + surrogate: Surrogate::new(91), + fields: vec![("body".to_string(), "hello world".to_string())], + provenance: None, + }; + let (record_type, payload) = crate::control::server::wal_dispatch::encode_text_op_record(&op) + .expect("encode text op") + .expect("an index op journals a record"); + let redo = RedoRecord { + version: 1, + ops: vec![RedoSubRecord { + record_type: record_type as u32, + payload, + }], + calvin_stamp: None, + cross_shard_applied: None, + row_sources: Vec::new(), + publishes: Vec::new(), + row_changes: Vec::new(), + } + .to_bytes() + .expect("encode redo"); + let plans = [PhysicalPlan::Text(op)]; + + let mut core = fresh_core(); + install( + &mut core.core, + redo, + &plans, + CalvinReplySpec::Count(Vec::new()), + ); + + assert_parity(&core.core, &plans); +} diff --git a/nodedb/src/data/executor/vector_checkpoint/write.rs b/nodedb/src/data/executor/vector_checkpoint/write.rs index c2b5d91b0..5324a51fe 100644 --- a/nodedb/src/data/executor/vector_checkpoint/write.rs +++ b/nodedb/src/data/executor/vector_checkpoint/write.rs @@ -330,6 +330,7 @@ mod tests { PhysicalPlan::Meta(MetaOp::WalAppend { payload: Vec::new(), }), + None, ); let response = restored.execute_multi_vector_delete(&task, 7, "docs", "emb", one_vector); assert_eq!(response.status, Status::Ok); diff --git a/nodedb/src/data/executor/wal_replay_redo_document.rs b/nodedb/src/data/executor/wal_replay_redo_document.rs index 0d2cf8fef..20d2e70ae 100644 --- a/nodedb/src/data/executor/wal_replay_redo_document.rs +++ b/nodedb/src/data/executor/wal_replay_redo_document.rs @@ -94,14 +94,6 @@ //! their targets and link their hash chain, exactly as replication does (see //! `handlers::transaction::redo_apply`). With no scope open this arm is plain //! restart replay. -//! -//! ### Calvin records in restart replay -//! -//! A Calvin redo record's stamp carries the sum targets its slice folds, and -//! no later record carries the target rows. Restart replay runs its document -//! rows through the committed path, which folds them at the record's LSN. -//! The fold subtracts the row's prior image, so a row that already holds its -//! post-image folds nothing. use nodedb_types::Surrogate; use nodedb_types::sync::wire::SyncProvenance; @@ -236,7 +228,7 @@ impl CoreLoop { self.observe_bitemporal_stamp(s.sys_from_ms); self.apply_scope.bitemporal_stamps.insert(surrogate_u32, s); } - let folds = self.redo_folds_at(record_lsn, &collection); + let folds = self.redo_folds(); let applied = if folds { self.apply_committed_document_put( CommittedDocWrite { @@ -335,7 +327,7 @@ impl CoreLoop { }, ); } - let folds = self.redo_folds_at(record_lsn, &collection); + let folds = self.redo_folds(); let removed = if folds { self.apply_committed_document_delete(CommittedDocWrite { database_id, @@ -380,18 +372,11 @@ impl CoreLoop { } } - /// Whether a document write at `record_lsn` to `collection` runs the - /// committed path, which folds materialized sums: always under a - /// committed-redo apply, and in restart replay when the record's Calvin - /// stamp names sum targets for `collection`. Folding reads the row's - /// prior image, so a replay over a row that already holds the post-image - /// folds nothing. - fn redo_folds_at(&self, record_lsn: u64, collection: &str) -> bool { + /// Whether a document write runs the committed path, which folds + /// materialized sums: only under a committed-redo apply. Restart replay + /// applies the fold target rows from the parts the apply journalled. + fn redo_folds(&self) -> bool { self.redo_apply.scope.is_some() - || self - .redo_apply - .replay_folds_for(record_lsn, collection) - .is_some() } } diff --git a/nodedb/src/data/executor/wal_replay_vector.rs b/nodedb/src/data/executor/wal_replay_vector.rs index 25ff6011c..28aec74ed 100644 --- a/nodedb/src/data/executor/wal_replay_vector.rs +++ b/nodedb/src/data/executor/wal_replay_vector.rs @@ -198,6 +198,7 @@ impl CoreLoop { pk_bytes: None, provenance: provenance.clone(), }), + Self::replay_record_lsn(record_lsn), ); let response = self.execute_vector_insert( crate::data::executor::handlers::vector::VectorInsertParams { diff --git a/nodedb/src/data/executor/wal_replay_vector_delete.rs b/nodedb/src/data/executor/wal_replay_vector_delete.rs index c705f410b..bf3ddcb57 100644 --- a/nodedb/src/data/executor/wal_replay_vector_delete.rs +++ b/nodedb/src/data/executor/wal_replay_vector_delete.rs @@ -89,6 +89,7 @@ impl CoreLoop { provenance: provenance.clone(), }, ), + Self::replay_record_lsn(record_lsn), ); let response = self.execute_vector_delete_by_surrogate( &task, diff --git a/nodedb/src/data/executor/wal_replay_vector_direct.rs b/nodedb/src/data/executor/wal_replay_vector_direct.rs index 86cfd7b54..a89d79d8d 100644 --- a/nodedb/src/data/executor/wal_replay_vector_direct.rs +++ b/nodedb/src/data/executor/wal_replay_vector_direct.rs @@ -81,6 +81,7 @@ impl CoreLoop { rls_filters: Vec::new(), rls_write_check: nodedb_types::RlsWriteCheck::already_decided_elsewhere(), }), + Self::replay_record_lsn(record_lsn), ); let response = self.execute_vector_direct_delete( crate::data::executor::handlers::vector_direct_delete::VectorDirectDeleteParams { @@ -162,6 +163,7 @@ impl CoreLoop { field: field.clone(), restart_identity: false, }), + Self::replay_record_lsn(record_lsn), ); let response = self.execute_vector_direct_truncate(&task, tenant_id, &collection, &field); if response.status != Status::Ok { @@ -250,6 +252,7 @@ impl CoreLoop { rls_filters: Vec::new(), rls_write_check: nodedb_types::RlsWriteCheck::already_decided_elsewhere(), }), + Self::replay_record_lsn(record_lsn), ); let response = self.execute_vector_direct_update( crate::data::executor::handlers::vector_direct_update::VectorDirectUpdateParams { diff --git a/nodedb/src/data/executor/wal_replay_vector_extended.rs b/nodedb/src/data/executor/wal_replay_vector_extended.rs index a58bed7f2..30b0ea471 100644 --- a/nodedb/src/data/executor/wal_replay_vector_extended.rs +++ b/nodedb/src/data/executor/wal_replay_vector_extended.rs @@ -253,6 +253,7 @@ impl CoreLoop { on_conflict_updates: on_conflict_updates.clone(), rls_write_check: nodedb_types::RlsWriteCheck::already_decided_elsewhere(), }), + Self::replay_record_lsn(record_lsn), ); let response = self.execute_vector_direct_upsert( crate::data::executor::handlers::vector_upsert::VectorDirectUpsertParams { @@ -351,6 +352,7 @@ impl CoreLoop { count, dim, }), + Self::replay_record_lsn(record_lsn), ); let response = self.execute_multi_vector_insert( crate::data::executor::handlers::vector_multi::MultiVectorInsertParams { @@ -440,6 +442,7 @@ impl CoreLoop { field_name: field_name.clone(), document_surrogate, }), + Self::replay_record_lsn(record_lsn), ); // The delete is idempotent: an already-absent document answers Ok. // Advance the watermark regardless so a later checkpoint records the diff --git a/nodedb/src/data/executor/wal_replay_vector_resolved.rs b/nodedb/src/data/executor/wal_replay_vector_resolved.rs index c396686d0..eafbc25d5 100644 --- a/nodedb/src/data/executor/wal_replay_vector_resolved.rs +++ b/nodedb/src/data/executor/wal_replay_vector_resolved.rs @@ -94,6 +94,7 @@ impl CoreLoop { response_payload: Vec::new(), rls_write_check: rls_write_check.clone(), }), + Self::replay_record_lsn(record_lsn), ); let index = VectorResolvedIndexSpec { collection: &collection, diff --git a/nodedb/src/data/executor/wal_replay_vector_sparse.rs b/nodedb/src/data/executor/wal_replay_vector_sparse.rs index a09953701..280d2082e 100644 --- a/nodedb/src/data/executor/wal_replay_vector_sparse.rs +++ b/nodedb/src/data/executor/wal_replay_vector_sparse.rs @@ -100,6 +100,7 @@ impl CoreLoop { doc_id: doc_id.clone(), entries: entries.clone(), }), + Self::replay_record_lsn(record_lsn), ); let response = self.execute_sparse_insert( &task, @@ -170,6 +171,7 @@ impl CoreLoop { field_name: field_name.clone(), doc_id: doc_id.clone(), }), + Self::replay_record_lsn(record_lsn), ); // An absent document yields NotFound; that is an expected idempotent // no-op on replay, not a failure. diff --git a/nodedb/src/data/executor/wal_replay_vector_task.rs b/nodedb/src/data/executor/wal_replay_vector_task.rs index 71d4aa1bc..3f6995c45 100644 --- a/nodedb/src/data/executor/wal_replay_vector_task.rs +++ b/nodedb/src/data/executor/wal_replay_vector_task.rs @@ -9,16 +9,26 @@ use crate::types::{DatabaseId, ReadConsistency}; use super::core_loop::CoreLoop; impl CoreLoop { + /// The LSN a replayed record hands its task. A `record_lsn` of 0 means + /// the record has no durable LSN and yields `None`. + pub(in crate::data::executor) fn replay_record_lsn( + record_lsn: u64, + ) -> Option { + (record_lsn != 0).then(|| crate::types::Lsn::new(record_lsn)) + } + /// Build a synthetic `ExecutionTask` for WAL replay. /// /// Mirrors `CoreLoop::replay_task` (`replay_task.rs`). The task carries - /// no meaningful request semantics — it is only needed so that the handler - /// methods can return a typed `Response`. + /// no meaningful request semantics. It lets the handler methods return a + /// typed `Response`. The task carries the record's LSN so the handler + /// records the write's version. pub(in crate::data::executor) fn replay_vector_task( tenant_id: crate::types::TenantId, database_id: DatabaseId, vshard_id: crate::types::VShardId, plan: PhysicalPlan, + wal_lsn: Option, ) -> ExecutionTask { ExecutionTask { request: Request { @@ -38,7 +48,7 @@ impl CoreLoop { user_id: None, statement_digest: None, txn_id: None, - wal_lsn: None, + wal_lsn, resolved_now_ms: None, commit_hlc: None, admission: crate::bridge::envelope::Admission::Exempt( @@ -46,7 +56,7 @@ impl CoreLoop { ), }, state: TaskState::Running, - wal_lsn: None, + wal_lsn, resolved_now_ms: None, } } diff --git a/nodedb/src/diag/context/data_plane.rs b/nodedb/src/diag/context/data_plane.rs index ef97848d4..f1859a5ee 100644 --- a/nodedb/src/diag/context/data_plane.rs +++ b/nodedb/src/diag/context/data_plane.rs @@ -109,9 +109,9 @@ pub(in crate::diag) struct CalvinApplyHalted<'a> { pub vshard_id: u32, pub epoch: u64, pub position: u32, - /// Halt reason label (`dispatch_refused`, `flush_failed`, ...). + /// Halt reason label (`dispatch_refused`, `redo_apply_failed`, ...). pub reason: &'a str, - /// Sub-operation of the txn that failed (`stage`, `flush`, ...). + /// Sub-operation of the txn that failed (`stage`, `redo_propose`, ...). pub step: &'a str, pub error: &'a str, } diff --git a/nodedb/src/diag/context/mod.rs b/nodedb/src/diag/context/mod.rs index 01f169304..d11059891 100644 --- a/nodedb/src/diag/context/mod.rs +++ b/nodedb/src/diag/context/mod.rs @@ -51,7 +51,7 @@ pub(in crate::diag) use quota::{ QuotaRowNotInstalled, QuotaRowWriteFailed, QuotaScopePurgeIncomplete, QuotaScopeReplayAborted, ScopeQuotaNotInstalled, }; -pub(in crate::diag) use raft_apply::{RaftEntryReapplied, ReplicatedWriteParked}; +pub(in crate::diag) use raft_apply::RaftEntryReapplied; pub(in crate::diag) use recovery::{ReplayRecordUnapplied, WalArchivalFailedTruncationHeld}; pub(in crate::diag) use redo_stream::{ RedoAbandonGivenUp, RedoSnapshotDebtNotRecorded, RedoStreamLostHere, diff --git a/nodedb/src/diag/context/raft_apply.rs b/nodedb/src/diag/context/raft_apply.rs index e3f06c015..892b76af7 100644 --- a/nodedb/src/diag/context/raft_apply.rs +++ b/nodedb/src/diag/context/raft_apply.rs @@ -40,40 +40,3 @@ impl DomainContext for RaftEntryReapplied { }) } } - -/// A replicated write waited on a core for a staged Calvin transaction that -/// owns its rows. The data group's apply loop awaits it, so every later entry -/// of every group waits too. -pub(in crate::diag) struct ReplicatedWriteParked { - pub core_id: usize, - pub collection: String, - pub owner_epoch: u64, - pub owner_position: u32, - pub owner_vshard: u32, -} - -impl DomainContext for ReplicatedWriteParked { - fn domain_kind(&self) -> &'static str { - "nodedb.replicated_write_parked" - } - - fn grouping_key(&self) -> String { - "replicated_write_parked".to_string() - } - - fn to_json(&self) -> Value { - json!({ - "core_id": self.core_id, - "collection": self.collection, - "owner_epoch": self.owner_epoch, - "owner_position": self.owner_position, - "owner_vshard": self.owner_vshard, - "why_fatal": "the data-group apply loop applies committed entries one at a \ - time. A parked replicated write holds it until the Calvin \ - transaction flushes or drops, so replicated writes to unrelated \ - collections wait behind an external event", - "operator_action": "check the named Calvin transaction's flush; the parked \ - write is refused at its request deadline", - }) - } -} diff --git a/nodedb/src/diag/mod.rs b/nodedb/src/diag/mod.rs index 6f77c4f82..7d8dcae58 100644 --- a/nodedb/src/diag/mod.rs +++ b/nodedb/src/diag/mod.rs @@ -21,11 +21,10 @@ pub use recording::{ quota_row_invalid, quota_row_undecodable, quota_row_write_failed, quota_scope_purge_incomplete, quota_scope_replay_aborted, raft_entries_reapplied, raft_entry_reapplied, redo_abandon_given_up, redo_snapshot_debt_not_recorded, redo_stream_lost_here, - replay_record_unapplied, replicated_write_parked, replicated_writes_parked, - retention_autowire_orphaned, scope_quota_not_installed, strict_row_image_unrendered, - strict_row_undecodable, synonym_group_not_applied, timeseries_partition_unreadable, - timeseries_row_image_undecodable, vector_build_failed, vector_builder_disconnected, - vector_builder_spawn_failed, vector_index_not_applied, vector_rebuild_unreadable, - wal_archival_failed_truncation_held, write_acked_without_durability, write_window_held, - write_window_leaked, + replay_record_unapplied, retention_autowire_orphaned, scope_quota_not_installed, + strict_row_image_unrendered, strict_row_undecodable, synonym_group_not_applied, + timeseries_partition_unreadable, timeseries_row_image_undecodable, vector_build_failed, + vector_builder_disconnected, vector_builder_spawn_failed, vector_index_not_applied, + vector_rebuild_unreadable, wal_archival_failed_truncation_held, write_acked_without_durability, + write_window_held, write_window_leaked, }; diff --git a/nodedb/src/diag/recording/mod.rs b/nodedb/src/diag/recording/mod.rs index e5130adae..ac1d2a388 100644 --- a/nodedb/src/diag/recording/mod.rs +++ b/nodedb/src/diag/recording/mod.rs @@ -50,9 +50,7 @@ pub use quota::{ quota_row_invalid, quota_row_undecodable, quota_row_write_failed, quota_scope_purge_incomplete, quota_scope_replay_aborted, scope_quota_not_installed, }; -pub use raft_apply::{ - raft_entries_reapplied, raft_entry_reapplied, replicated_write_parked, replicated_writes_parked, -}; +pub use raft_apply::{raft_entries_reapplied, raft_entry_reapplied}; pub use recovery::{ batch_insert_without_surrogates, fts_index_update_failed, orphaned_index_entry_after_delete, replay_record_unapplied, strict_row_undecodable, wal_archival_failed_truncation_held, diff --git a/nodedb/src/diag/recording/raft_apply.rs b/nodedb/src/diag/recording/raft_apply.rs index 55152d53e..acf338aa3 100644 --- a/nodedb/src/diag/recording/raft_apply.rs +++ b/nodedb/src/diag/recording/raft_apply.rs @@ -11,21 +11,12 @@ use crate::diag::context; /// Count of committed Raft entries the apply loop applied a second time. static RAFT_ENTRIES_REAPPLIED: AtomicU64 = AtomicU64::new(0); -/// Count of replicated writes parked behind a staged Calvin transaction. -static REPLICATED_WRITES_PARKED: AtomicU64 = AtomicU64::new(0); - /// Read the count of committed Raft entries applied twice. Exposed for the /// metrics exporter and tests. pub fn raft_entries_reapplied() -> u64 { RAFT_ENTRIES_REAPPLIED.load(Ordering::Relaxed) } -/// Read the count of replicated writes parked behind a staged Calvin -/// transaction. Exposed for the metrics exporter and tests. -pub fn replicated_writes_parked() -> u64 { - REPLICATED_WRITES_PARKED.load(Ordering::Relaxed) -} - /// Report an apply of a Raft log index the apply loop already applied. /// Called from the apply loop, the one site that sees every applied index. pub fn raft_entry_reapplied(group_id: u64, log_index: u64, highest_applied: u64) { @@ -43,23 +34,3 @@ pub fn raft_entry_reapplied(group_id: u64, log_index: u64, highest_applied: u64) .with_backtrace() .emit(); } - -/// Report a replicated write a core parked behind a staged Calvin -/// transaction. Called from the core's Calvin fence when it parks a write -/// whose order Raft already fixed. -pub fn replicated_write_parked(core_id: usize, collection: &str, owner: (u64, u32, u32)) { - REPLICATED_WRITES_PARKED.fetch_add(1, Ordering::Relaxed); - let ctx = context::ReplicatedWriteParked { - core_id, - collection: collection.to_owned(), - owner_epoch: owner.0, - owner_position: owner.1, - owner_vshard: owner.2, - }; - let _ = Capture::new( - EventKind::Error, - "replicated write parked behind a staged Calvin transaction", - ) - .domain(&ctx) - .emit(); -} diff --git a/nodedb/src/error_from_data_plane.rs b/nodedb/src/error_from_data_plane.rs index e8cd847ed..8e11b6993 100644 --- a/nodedb/src/error_from_data_plane.rs +++ b/nodedb/src/error_from_data_plane.rs @@ -55,6 +55,12 @@ pub(crate) fn data_plane_code_to_public(code: ErrorCode) -> NodeDbError { PublicCode::WRITE_CONFLICT, format!("write refused without applying, retry: {reason}"), ), + // The core applied nothing, and another replica or a restart serves + // the request: the retriable class. + ErrorCode::CoreFailStopped { core_id, detail } => NodeDbError::from_wire( + PublicCode::WRITE_CONFLICT, + format!("core {core_id} is fail-stopped and applied nothing, retry: {detail}"), + ), // The Data Plane cannot distinguish an absent collection from an // absent row through this code, and `document_not_found` is the // narrower of the two claims: it never asserts the collection is diff --git a/nodedb/src/wal/redo/record.rs b/nodedb/src/wal/redo/record.rs index 1de22dbd1..85b10cb9c 100644 --- a/nodedb/src/wal/redo/record.rs +++ b/nodedb/src/wal/redo/record.rs @@ -175,8 +175,9 @@ pub struct RedoSubRecord { pub payload: Vec, } -/// Calvin sequencer coordinates that a [`RedoRecord`] may carry to double as an -/// applied-marker, and what the vShard's slice folds. Mirrors +/// Calvin sequencer coordinates a [`RedoRecord`] carries when it installs a +/// committed Calvin slice. The record doubles as the position's applied +/// marker: boot recovery reads the stamp. Mirrors /// `nodedb_wal::CalvinAppliedPayload` in its coordinates. #[derive( Debug, @@ -196,17 +197,6 @@ pub struct CalvinStamp { pub position: u32, /// The vshard that applied this transaction. pub vshard_id: u32, - /// Every collection the vShard's slice writes. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - #[msgpack(default)] - pub collections: Vec, - /// The materialized-sum targets the slice's document writes fold into, - /// keyed by source collection. The live install and restart replay both - /// fold from them at the record's LSN, so no later record carries the - /// target rows. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - #[msgpack(default)] - pub sum_targets: Vec, } /// Payload of a graph edge upsert: the bytes of an autocommit @@ -470,8 +460,6 @@ mod tests { epoch: 42, position: 7, vshard_id: 3, - collections: Vec::new(), - sum_targets: Vec::new(), }), cross_shard_applied: None, row_sources: Vec::new(), diff --git a/nodedb/src/wal/redo/replay.rs b/nodedb/src/wal/redo/replay.rs index 2a70e607f..42361c99b 100644 --- a/nodedb/src/wal/redo/replay.rs +++ b/nodedb/src/wal/redo/replay.rs @@ -43,7 +43,9 @@ //! `replay_graph_redo` arms in `crate::data::executor`). //! //! `calvin_stamp` is ignored here: the Calvin recovery scan reads it, and it -//! does not gate engine replay. +//! does not gate engine replay. A Calvin slice's materialized-sum fold rows +//! journal as parts of the record's group, so replay applies them like any +//! other write's rows. //! //! The committed-redo apply (`handlers::transaction::redo_apply`) drives this //! same entry point with a one-record slice for every committed transaction, @@ -85,20 +87,12 @@ use crate::data::executor::core_loop::CoreLoop; /// on its core. Replay then applies a write's rows at the write's place in /// the order, wherever its parts sit in the WAL: a part boot journals at the /// tail never lands over a later write to the same row. -/// -/// Also returns the materialized-sum targets every Calvin record's stamp -/// carries, keyed by the record's LSN. -fn reconstitute_with_folds(records: &[WalRecord]) -> crate::Result { - let mut out = Reconstituted::default(); +fn reconstitute_redo_records(records: &[WalRecord]) -> crate::Result> { + let mut out = Vec::new(); for record in records { let (lsn, ops) = match RecordType::from_raw(record.logical_record_type()) { Some(RecordType::TransactionRedo) => { let redo = RedoRecord::from_bytes(&record.payload)?; - if let Some(stamp) = redo.calvin_stamp - && !stamp.sum_targets.is_empty() - { - out.folds.insert(record.header.lsn, stamp.sum_targets); - } (record.header.lsn, redo.ops) } Some(RecordType::WriteGroup) => { @@ -108,7 +102,7 @@ fn reconstitute_with_folds(records: &[WalRecord]) -> crate::Result continue, }; for sub in ops { - out.ops.push(WalRecord::new(WalRecordArgs { + out.push(WalRecord::new(WalRecordArgs { record_type: sub.record_type, lsn, tenant_id: record.header.tenant_id, @@ -123,18 +117,6 @@ fn reconstitute_with_folds(records: &[WalRecord]) -> crate::Result, - folds: RedoFolds, -} - -/// Materialized-sum targets per redo record LSN. -type RedoFolds = - std::collections::HashMap>; - /// Merge the standalone records with the reconstituted redo sub-records into /// one LSN-ordered sequence. /// @@ -198,25 +180,15 @@ impl CoreLoop { num_cores: usize, tombstones: &nodedb_wal::TombstoneSet, ) -> crate::Result<()> { - let Reconstituted { - ops: redo_ops, - folds, - } = reconstitute_with_folds(records)?; + let redo_ops = reconstitute_redo_records(records)?; let ordered = merge_by_lsn(records, &redo_ops); - // A committed-redo apply folds from its open scope. Restart replay - // folds from each Calvin record's stamp. - let restart = self.begin_replay_folds(folds); self.replay_vector_wal(&ordered, num_cores, tombstones); crate::fail_point!("replay::between_engine_passes"); self.replay_vector_extended_wal(&ordered, num_cores, tombstones); // Every document row the WAL names, standalone and redo alike, in LSN - // order. A Calvin record's rows fold into their sum targets at the - // record's LSN, so the fold scope closes right after this arm. + // order. self.replay_document_redo(&ordered, num_cores, tombstones); - if restart { - self.end_replay_folds(); - } self.replay_kv_wal(&ordered, num_cores, tombstones); self.replay_timeseries_wal(&ordered, num_cores, tombstones); self.replay_array_wal(&ordered, num_cores, tombstones); @@ -271,11 +243,6 @@ mod tests { use super::*; use crate::wal::{RedoRecord, RedoSubRecord}; - /// The reconstituted records alone, without the fold targets. - fn reconstitute_redo_records(records: &[WalRecord]) -> crate::Result> { - Ok(super::reconstitute_with_folds(records)?.ops) - } - fn redo_wal_record(lsn: u64, tenant_id: u64, vshard_id: u32, record: &RedoRecord) -> WalRecord { WalRecord::new(WalRecordArgs { record_type: RecordType::TransactionRedo as u32, diff --git a/nodedb/tests/calvin_hold_liveness.rs b/nodedb/tests/calvin_hold_liveness.rs index 699e140d9..af52f6eff 100644 --- a/nodedb/tests/calvin_hold_liveness.rs +++ b/nodedb/tests/calvin_hold_liveness.rs @@ -1,12 +1,14 @@ // SPDX-License-Identifier: BUSL-1.1 -//! A held Calvin flush never stalls replicated writes to other collections. +//! A held Calvin redo proposal never stalls replicated writes to other +//! collections. //! //! Transaction A writes two collections on two vShards, so it commits -//! through the Calvin scheduler. Its flush on one vShard is held at a fail -//! point, so A stays staged: its rows are owned until the flush. A replicated -//! autocommit write to a third collection shares no row and no collection -//! with A. It must apply and be acknowledged while A's flush is held. +//! through the Calvin scheduler. The redo proposal of its slice on one vShard +//! is held at a fail point, so A stays staged and holds its locks. A +//! replicated autocommit write to a third collection shares no row and no +//! collection with A. It must apply and be acknowledged while A's redo is +//! held. //! //! Requires `--features failpoints`. @@ -23,27 +25,27 @@ use crash_harness::vshards::names_on_distinct_vshards; /// How long the Calvin sequencer may take to elect its leader after a boot. const CALVIN_READY_TIMEOUT: Duration = Duration::from_secs(30); -/// How long the test waits for A's flush to be held. +/// How long the test waits for A's redo proposal to be held. const HOLD_DEADLINE: Duration = Duration::from_secs(30); /// How long the unrelated write may take. Well under the 30s request -/// deadline a write parked behind A would wait out. +/// deadline a write waiting behind A would wait out. const UNRELATED_WRITE_BUDGET: Duration = Duration::from_secs(10); -/// The scheduler's log line for a flush held at the fail point. -const FLUSH_HELD: &str = "calvin: flush held at a fail point"; +/// The scheduler's log line for a redo proposal held at the fail point. +const REDO_HELD: &str = "calvin: redo proposal held at a fail point"; const LOG_DIRECTIVES: &str = "warn,nodedb::control::cluster::calvin::scheduler::driver::core=info"; #[tokio::test(flavor = "multi_thread")] -async fn a_replicated_write_to_an_unrelated_collection_applies_while_a_calvin_flush_is_held() { +async fn a_replicated_write_to_an_unrelated_collection_applies_while_a_calvin_redo_is_held() { let [held, peer, other] = names_on_distinct_vshards(["hold_held", "hold_peer", "hold_other"]); let h = CrashHarness::new(); - let release = h.data_dir().join("release-held-flush"); + let release = h.data_dir().join("release-held-redo"); let mut h = h.with_env("RUST_LOG", LOG_DIRECTIVES).with_env( "NODEDB_FAILPOINTS", &format!( - "calvin::before_flush::{held}=wait_file({})", + "calvin::before_redo_propose::{held}=wait_file({})", release.display() ), ); @@ -58,7 +60,7 @@ async fn a_replicated_write_to_an_unrelated_collection_applies_while_a_calvin_fl } // Transaction A, on its own connection: its COMMIT waits for the held - // flush. + // redo. let conn_str = h.pgwire_conn_str(); let statements = [ "BEGIN".to_string(), @@ -81,10 +83,10 @@ async fn a_replicated_write_to_an_unrelated_collection_applies_while_a_calvin_fl }); let deadline = Instant::now() + HOLD_DEADLINE; - while count_lines(&boot_section(&h.server_log(), 1), &[FLUSH_HELD]) == 0 { + while count_lines(&boot_section(&h.server_log(), 1), &[REDO_HELD]) == 0 { assert!( Instant::now() < deadline && !txn.is_finished(), - "transaction A's flush was never held (A finished: {}).{}\n{}", + "transaction A's redo was never held (A finished: {}).{}\n{}", txn.is_finished(), h.keep_data_dir_note(), crash_harness::diagnostics::log_tail_section(&h.server_log()) @@ -99,7 +101,7 @@ async fn a_replicated_write_to_an_unrelated_collection_applies_while_a_calvin_fl .await .unwrap_or_else(|_| { panic!( - "a replicated write to {other} stalled behind transaction A's held flush, \ + "a replicated write to {other} stalled behind transaction A's held redo, \ though it shares no row and no collection with A" ) }); diff --git a/nodedb/tests/crash_replay_stamp_calvin.rs b/nodedb/tests/crash_replay_stamp_calvin.rs index 7dc7dd643..d15ef09bf 100644 --- a/nodedb/tests/crash_replay_stamp_calvin.rs +++ b/nodedb/tests/crash_replay_stamp_calvin.rs @@ -1,29 +1,29 @@ // SPDX-License-Identifier: BUSL-1.1 -//! A Calvin transaction's redo record still in flight when a checkpoint is -//! written survives a crash. +//! A Calvin slice's stamped redo still in flight when a checkpoint is written +//! survives a crash. //! //! The server runs the default single-node Calvin stack. A transaction that -//! writes two collections on different vShards commits through the Calvin -//! scheduler. Each vShard's scheduler appends the transaction's redo record, -//! then sends a flush that installs it on the core. The sequence: +//! writes two collections in different data groups commits through the Calvin +//! scheduler. On the COMMIT verdict each slice's leader proposes the slice's +//! stamped redo to its data group, and the group's apply installs it. The +//! sequence: //! -//! 1. Transaction A writes the held collection and a peer collection on -//! another vShard. The held vShard's scheduler appends A's redo record and -//! holds its flush at a fail point. No core installed the record. -//! 2. Writes B, autocommit `INSERT`s into a third collection, apply with -//! higher LSNs until a KV checkpoint's replay stamp names one of them above -//! its prefix. That proves the checkpoint was written while A's record was -//! appended and not applied. +//! 1. Transaction A writes the held collection and a peer collection. The +//! held group's apply appends A's stamped redo record and parks it at +//! `funnel::before_dispatch::`. No core installed the record. +//! 2. Writes B, autocommit `INSERT`s into a third collection in another data +//! group, apply with higher LSNs until a KV checkpoint's replay stamp names +//! one of them above its prefix. That proves the checkpoint was written +//! while A's record was appended and not applied. //! A's COMMIT stays unacknowledged for the whole hold. It either still //! waits, or it gave up at the server's completion deadline, which is no -//! acknowledgement. A COMMIT that returned success is the bug the hold -//! guards against. -//! 3. The test releases the flush. The core installs A's record, and the -//! process aborts before the flush response leaves. -//! 4. After restart, replay must apply A's record. The Calvin scheduler reads -//! the same record as the transaction's applied marker and never runs the -//! transaction again, so replay is the only way A's row returns. +//! acknowledgement. +//! 3. The test releases the gate. The core installs A's record, and the +//! process aborts before the install's response leaves. +//! 4. After restart, replay must apply A's record. Its stamp marks A's +//! position applied, so the scheduler never runs A again and the data +//! group's re-delivered entry installs nothing. //! //! Requires `--features failpoints`. @@ -33,47 +33,39 @@ mod crash_harness; use std::time::{Duration, Instant}; -use crash_harness::log_fields::{boot_section, count_lines, log_field}; -use crash_harness::vshards::names_on_distinct_vshards; +use crash_harness::log_fields::{boot_section, log_field}; +use crash_harness::vshards::names_in_distinct_data_groups; use crash_harness::{CrashHarness, diagnostics}; -/// How long the test waits for the held flush, and for a checkpoint whose +/// How long the test waits for the held install, and for a checkpoint whose /// stamp names a B: thirty checkpoint cycles at one per second. const CHECKPOINT_DEADLINE: Duration = Duration::from_secs(30); -/// How long the process may take to abort once the flush is released. The -/// scheduler sends a held flush again on its next pass. +/// How long the process may take to abort once the gate is released. const CRASH_TIMEOUT: Duration = Duration::from_secs(60); /// How long the Calvin sequencer may take to elect its leader after a boot. const CALVIN_READY_TIMEOUT: Duration = Duration::from_secs(30); -/// The scheduler's log line for a flush held at the fail point. -const FLUSH_HELD: &str = "calvin: flush held at a fail point"; - /// The error a COMMIT returns once its completion deadline passes with no -/// acknowledgement. The redo record stays appended and the flush stays held. +/// acknowledgement. The redo record stays appended and the install stays held. const COMMIT_DEADLINE: &str = "timed out waiting for Calvin transaction completion"; -const LOG_DIRECTIVES: &str = "warn,nodedb::data::executor::kv_checkpoint=info,\ - nodedb::control::cluster::calvin::scheduler::driver::core=info"; +const LOG_DIRECTIVES: &str = "warn,nodedb::data::executor::kv_checkpoint=info"; + +/// The boot that holds A's install. +const HOLD_BOOT: u32 = 2; + +/// The boot after the crash. +const RESTORE_BOOT: u32 = 3; #[tokio::test(flavor = "multi_thread")] async fn a_calvin_redo_in_flight_at_a_checkpoint_survives_kill_9() { let [held, peer, applied] = - names_on_distinct_vshards(["calvin_held", "calvin_peer", "calvin_applied"]); - let h = CrashHarness::new(); - let release = h.data_dir().join("release-held-flush"); - let mut h = h + names_in_distinct_data_groups(["calvin_held", "calvin_peer", "calvin_applied"]); + let mut h = CrashHarness::new() .with_env("NODEDB_CHECKPOINT_INTERVAL_SECS", "1") - .with_env("RUST_LOG", LOG_DIRECTIVES) - .with_env( - "NODEDB_FAILPOINTS", - &format!( - "calvin::before_flush::{held}=wait_file({}),core::after_apply::{held}=abort", - release.display() - ), - ); + .with_env("RUST_LOG", LOG_DIRECTIVES); h.spawn(); h.wait_ready(); h.wait_for_calvin_ready(CALVIN_READY_TIMEOUT).await; @@ -84,7 +76,23 @@ async fn a_calvin_redo_in_flight_at_a_checkpoint_survives_kill_9() { .await; } - // Transaction A. Its COMMIT waits for the held flush, so it runs on its + // The next boot arms the gate and the abort, keyed to the held + // collection. Both match only a request carrying a WAL LSN, so boot + // itself passes them. + h.kill_9(); + let release = h.data_dir().join("release-held-install"); + let parked = h.data_dir().join("release-held-install.parked"); + h.set_env( + "NODEDB_FAILPOINTS", + &format!( + "funnel::before_dispatch::{held}=wait_file({}),core::after_apply::{held}=abort", + release.display() + ), + ); + h.reopen(); + h.wait_for_calvin_ready(CALVIN_READY_TIMEOUT).await; + + // Transaction A. Its COMMIT waits for the held install, so it runs on its // own connection. let conn_str = h.pgwire_conn_str(); let statements = [ @@ -110,13 +118,13 @@ async fn a_calvin_redo_in_flight_at_a_checkpoint_survives_kill_9() { Ok::<(), String>(()) }); - // A's redo record is appended before its flush is sent, so a held flush - // proves the record is in the WAL and not installed. + // The gate parks a write after its record is appended, so a parked + // install proves A's stamped record is in the WAL and not installed. let deadline = Instant::now() + CHECKPOINT_DEADLINE; - while count_lines(&boot_section(&h.server_log(), 1), &[FLUSH_HELD]) == 0 { + while !parked.exists() { assert!( Instant::now() < deadline && !txn_task.is_finished(), - "the flush of transaction A was never held: A did not commit through the \ + "the install of transaction A was never held: A did not commit through the \ Calvin scheduler.{}\n{}", h.keep_data_dir_note(), diagnostics::log_tail_section(&h.server_log()) @@ -134,7 +142,7 @@ async fn a_calvin_redo_in_flight_at_a_checkpoint_survives_kill_9() { .await; written += 1; tokio::time::sleep(Duration::from_millis(200)).await; - let log = boot_section(&h.server_log(), 1); + let log = boot_section(&h.server_log(), HOLD_BOOT); if log_field(&log, "KV checkpoint published", "applied_ranges") .iter() .any(|n| *n > 0) @@ -144,7 +152,7 @@ async fn a_calvin_redo_in_flight_at_a_checkpoint_survives_kill_9() { assert!( Instant::now() < deadline, "no KV checkpoint named an applied LSN above its prefix within \ - {CHECKPOINT_DEADLINE:?} while A's flush was held.{}\n{}", + {CHECKPOINT_DEADLINE:?} while A's install was held.{}\n{}", h.keep_data_dir_note(), diagnostics::log_tail_section(&h.server_log()) ); @@ -157,7 +165,7 @@ async fn a_calvin_redo_in_flight_at_a_checkpoint_survives_kill_9() { match outcome { Err(error) if error.contains(COMMIT_DEADLINE) => None, other => panic!( - "transaction A finished before its flush was released: {other:?}.{}\n{}", + "transaction A finished before its install was released: {other:?}.{}\n{}", h.keep_data_dir_note(), diagnostics::log_tail_section(&h.server_log()) ), @@ -169,14 +177,14 @@ async fn a_calvin_redo_in_flight_at_a_checkpoint_survives_kill_9() { let mut live = h.query_col_idx(&read_applied, 0).await; live.sort(); - // Release the flush. The core installs A's record, and the process aborts - // before the flush response leaves. + // Release the gate. The core installs A's record, and the process aborts + // before the install's response leaves. std::fs::write(&release, b"release").expect("create the release file"); h.await_self_crash(CRASH_TIMEOUT); let marker = format!("fail_point aborting process: core::after_apply::{held}"); assert!( h.server_log().contains(&marker), - "the process exited, but not after A's flush installed its record.{}\n{}", + "the process exited, but not after A's install applied its record.{}\n{}", h.keep_data_dir_note(), diagnostics::log_tail_section(&h.server_log()) ); @@ -190,7 +198,7 @@ async fn a_calvin_redo_in_flight_at_a_checkpoint_survives_kill_9() { h.wait_for_calvin_ready(CALVIN_READY_TIMEOUT).await; let proof = log_field( - &boot_section(&h.server_log(), 2), + &boot_section(&h.server_log(), RESTORE_BOOT), "KV checkpoint restored", "applied_ranges", ); diff --git a/nodedb/tests/inproc/cases/calvin_executor_apply.rs b/nodedb/tests/inproc/cases/calvin_executor_apply.rs index d622399e8..cab7d9c4d 100644 --- a/nodedb/tests/inproc/cases/calvin_executor_apply.rs +++ b/nodedb/tests/inproc/cases/calvin_executor_apply.rs @@ -3,9 +3,10 @@ //! Data Plane apply and rollback coverage for a committed Calvin transaction. //! //! These drive the participant's Data Plane steps directly: stage -//! (`CalvinExecuteStatic`), resolve (`CalvinResolve`) and flush the resolved -//! redo record at its LSN (`CalvinFlush`) — the steps the scheduler dispatches -//! once the global verdict is commit. +//! (`CalvinExecuteStatic`), resolve (`CalvinResolve`), and the stamped install +//! of the resolved redo record at its LSN (`ApplyTransactionRedo` with a +//! `CalvinInstall`), which the data group's apply runs once the leader +//! proposed the slice's stamped redo. use std::sync::Arc; use std::time::{Duration, Instant}; @@ -16,7 +17,9 @@ use nodedb::data::executor::core_loop::CoreLoop; use nodedb::types::*; use nodedb::wal::{RedoRecord, RedoSubRecord}; use nodedb_bridge::buffer::{Consumer, Producer, RingBuffer}; -use nodedb_physical::physical_plan::{KvOp, MetaOp, PhysicalPlan}; +use nodedb_physical::physical_plan::{ + CalvinInstall, CalvinReplySpec, CalvinResolved, KvOp, MetaOp, PhysicalPlan, RedoOrigin, +}; use nodedb_types::{OrdinalClock, QualifiedCollection}; // ── Helpers ───────────────────────────────────────────────────────────────── @@ -117,14 +120,14 @@ fn kv_get(coll: &str, key: &[u8]) -> PhysicalPlan { } /// Stage `plans` as the Calvin transaction at `(epoch, 0)` and resolve it. -/// Returns the resolved redo record. +/// Returns the resolved redo record and the slice's reply spec. fn stage_and_resolve( core: &mut CoreLoop, tx: &mut Producer, rx: &mut Consumer, epoch: u64, plans: Vec, -) -> RedoRecord { +) -> (RedoRecord, CalvinReplySpec) { send_ok( core, tx, @@ -135,7 +138,6 @@ fn stage_and_resolve( tenant_id: TenantId::new(1), plans, epoch_system_ms: 0, - is_group_leader: true, versioned_reads: Vec::new(), body_plans: Vec::new(), }), @@ -146,27 +148,33 @@ fn stage_and_resolve( rx, PhysicalPlan::Meta(MetaOp::CalvinResolve { epoch, position: 0 }), ); - let answer: nodedb_physical::physical_plan::CalvinResolved = - zerompk::from_msgpack(&redo).expect("decode resolved answer"); - RedoRecord::from_bytes(&answer.redo).expect("decode resolved redo") + let answer: CalvinResolved = zerompk::from_msgpack(&redo).expect("decode resolved answer"); + let record = RedoRecord::from_bytes(&answer.redo).expect("decode resolved redo"); + (record, answer.reply) } -/// Flush `redo` as the Calvin transaction at `(epoch, 0)` with its record at +/// Install `redo` as the stamped slice at `(epoch, 0)` with its record at /// `lsn`. -fn flush( +fn install( core: &mut CoreLoop, tx: &mut Producer, rx: &mut Consumer, epoch: u64, - redo: &RedoRecord, + (redo, reply): &(RedoRecord, CalvinReplySpec), lsn: u64, ) -> nodedb::bridge::envelope::Response { - let mut request = make_request(PhysicalPlan::Meta(MetaOp::CalvinFlush { - epoch, - position: 0, + let mut request = make_request(PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { redo: redo.to_bytes().expect("encode redo"), collections: vec!["orders".into(), "rollback_coll".into()], sum_targets: Vec::new(), + origin: RedoOrigin::Commit, + calvin: Some(CalvinInstall { + epoch, + position: 0, + epoch_system_ms: 0, + reply: reply.clone(), + user_write: true, + }), })); request.wal_lsn = Some(Lsn::new(lsn)); tx.try_push(BridgeRequest::unfloored(request)).unwrap(); @@ -197,9 +205,9 @@ fn failing_install_sub_record() -> RedoSubRecord { } } -// ── Test 1: successful Calvin flush ────────────────────────────────────────── +// ── Test 1: successful Calvin install ──────────────────────────────────────── -/// A committed Calvin transaction installs its redo record at the flush. The +/// A committed Calvin transaction installs its stamped redo record. The /// written key is readable after it. #[test] fn calvin_static_apply_success() { @@ -212,24 +220,24 @@ fn calvin_static_apply_success() { 1, vec![kv_put("orders", b"k1", b"v1")], ); - let resp = flush(&mut core, &mut tx, &mut rx, 1, &redo, 10); + let resp = install(&mut core, &mut tx, &mut rx, 1, &redo, 10); assert_eq!( resp.status, Status::Ok, - "Calvin flush must succeed; got {:?}", + "Calvin install must succeed; got {:?}", resp.error_code ); let payload = send_ok(&mut core, &mut tx, &mut rx, kv_get("orders", b"k1")); assert!( !payload.is_empty(), - "row must exist after a successful Calvin flush" + "row must exist after a successful Calvin install" ); } // ── Test 2: failing install → error without RollbackFailed ─────────────────── -/// When a sub-record of the redo record fails while it installs, the flush +/// When a sub-record of the redo record fails while it installs, the install /// rolls every write back. The response is `Status::Error` and is NOT /// `RollbackFailed` (the rollback itself succeeded), and the transaction's /// write is gone. @@ -244,13 +252,13 @@ fn calvin_static_apply_failure_rolls_back_cleanly() { 2, vec![kv_put("rollback_coll", b"should_be_gone", b"present")], ); - redo.ops.push(failing_install_sub_record()); - let resp = flush(&mut core, &mut tx, &mut rx, 2, &redo, 20); + redo.0.ops.push(failing_install_sub_record()); + let resp = install(&mut core, &mut tx, &mut rx, 2, &redo, 20); assert_eq!( resp.status, Status::Error, - "a failing sub-record must fail the flush; got {:?}", + "a failing sub-record must fail the install; got {:?}", resp.error_code ); assert!( diff --git a/nodedb/tests/inproc/cases/calvin_executor_panic_recovery.rs b/nodedb/tests/inproc/cases/calvin_executor_panic_recovery.rs index 9c1d3cc34..095cc11e6 100644 --- a/nodedb/tests/inproc/cases/calvin_executor_panic_recovery.rs +++ b/nodedb/tests/inproc/cases/calvin_executor_panic_recovery.rs @@ -1,10 +1,10 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Executor panic-recovery test for a committed Calvin transaction's flush. +//! Executor panic-recovery test for a committed Calvin slice's stamped install. //! -//! Compiled only with `--features failpoints`. Tests the flush's -//! panic-recovery path: when a panic fires while the flush installs the -//! transaction's redo record (`replay::between_standalone_and_redo`, after +//! Compiled only with `--features failpoints`. Tests the install's +//! panic-recovery path: when a panic fires while the stamped install writes +//! the transaction's redo record (`replay::between_standalone_and_redo`, after //! the KV writes landed), the install must: //! //! - Catch the panic and roll back every write the install made. @@ -13,12 +13,12 @@ //! //! ## Failure model alignment //! -//! Per the Calvin failure model: a panic while the flush installs causes the -//! shard to return `Status::Error`. The lock-manager invariant ("locks NOT -//! released on a failed flush") is enforced by the scheduler layer, which -//! halts. The executor's contract is: +//! Per the Calvin failure model: a panic while the stamped install writes +//! causes the shard to return `Status::Error`. The apply loop reports the +//! refusal, and the scheduler halts with the txn's locks held. The executor's +//! contract is: //! -//! 1. Rolled-back writes are not visible after the failed flush. +//! 1. Rolled-back writes are not visible after the failed install. //! 2. The error response is a retryable refusal naming the panic. //! 3. Subsequent operations on the same `CoreLoop` succeed (no state //! corruption from the unwind). @@ -43,7 +43,9 @@ use nodedb::bridge::envelope::{ErrorCode, Status}; #[cfg(feature = "failpoints")] use nodedb::fail_point::{FailAction, FailGuard}; #[cfg(feature = "failpoints")] -use nodedb_physical::physical_plan::{KvOp, MetaOp, PhysicalPlan}; +use nodedb_physical::physical_plan::{ + CalvinInstall, CalvinResolved, KvOp, MetaOp, PhysicalPlan, RedoOrigin, +}; #[cfg(feature = "failpoints")] use nodedb_types::TenantId as NodedbTenantId; #[cfg(feature = "failpoints")] @@ -60,7 +62,6 @@ fn calvin_static(epoch: u64, plans: Vec) -> PhysicalPlan { tenant_id: NodedbTenantId::new(1), plans, epoch_system_ms: 1_700_000_000_000, - is_group_leader: true, versioned_reads: vec![], body_plans: vec![], }) @@ -71,11 +72,11 @@ fn calvin_static(epoch: u64, plans: Vec) -> PhysicalPlan { const INSTALL_FAIL_POINT: &str = "replay::between_standalone_and_redo"; /// Stage a static Calvin transaction (validate + stage), resolve it into its -/// redo record, and flush the record at `lsn`, returning the flush response -/// (where any install-time panic surfaces). The stage and resolve steps must -/// always return `Status::Ok`. +/// redo record, and install the stamped record at `epoch * 10`, returning the +/// install response (where any install-time panic surfaces). The stage and +/// resolve steps must always return `Status::Ok`. #[cfg(feature = "failpoints")] -fn stage_then_flush( +fn stage_then_install( core: &mut nodedb::data::executor::core_loop::CoreLoop, tx: &mut nodedb_bridge::buffer::Producer, rx: &mut nodedb_bridge::buffer::Consumer, @@ -101,16 +102,20 @@ fn stage_then_flush( "resolve must succeed; got {:?}", resolved.error_code ); - let mut request = make_request(PhysicalPlan::Meta(MetaOp::CalvinFlush { - epoch, - position: 0, - redo: zerompk::from_msgpack::( - resolved.payload.as_bytes(), - ) - .expect("decode resolved answer") - .redo, + let answer: CalvinResolved = + zerompk::from_msgpack(resolved.payload.as_bytes()).expect("decode resolved answer"); + let mut request = make_request(PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { + redo: answer.redo, collections: Vec::new(), sum_targets: Vec::new(), + origin: RedoOrigin::Commit, + calvin: Some(CalvinInstall { + epoch, + position: 0, + epoch_system_ms: 1_700_000_000_000, + reply: answer.reply, + user_write: true, + }), })); request.wal_lsn = Some(nodedb::types::Lsn::new(epoch * 10)); tx.try_push(BridgeRequest::unfloored(request)).unwrap(); @@ -161,7 +166,8 @@ fn kv_get_in(coll: &str, key: &[u8]) -> PhysicalPlan { // ── Test 1: install panic caught, typed response returned ───────────────────── -/// Panic injected while the flush installs the transaction's redo record. +/// Panic injected while the stamped install writes the transaction's redo +/// record. /// /// The record carries two KV puts, which land before the fail point fires. /// The install must catch the unwind, roll both writes back, and answer with @@ -173,8 +179,8 @@ fn calvin_static_panic_returns_internal_error() { let _guard = FailGuard::install(INSTALL_FAIL_POINT, FailAction::Panic); - // Stage and resolve write nothing; the panic fires in the flush's install. - let resp = stage_then_flush( + // Stage and resolve write nothing; the panic fires in the install. + let resp = stage_then_install( &mut core, &mut tx, &mut rx, @@ -189,7 +195,7 @@ fn calvin_static_panic_returns_internal_error() { // ── Test 2: rolled-back writes not visible after Calvin panic ───────────────── -/// After a Calvin executor panic, writes from the failed flush must not be +/// After a Calvin executor panic, writes from the failed install must not be /// visible. The CoreLoop remains functional for subsequent requests. #[cfg(feature = "failpoints")] #[test] @@ -207,7 +213,7 @@ fn calvin_static_panic_rollback_not_visible() { // Inject a panic on the second sub-apply. let _guard = FailGuard::install(INSTALL_FAIL_POINT, FailAction::Panic); - let resp = stage_then_flush( + let resp = stage_then_install( &mut core, &mut tx, &mut rx, @@ -220,7 +226,7 @@ fn calvin_static_panic_rollback_not_visible() { assert_eq!( resp.status, Status::Error, - "a panicking flush must return Error; got {:?}", + "a panicking install must return Error; got {:?}", resp.status ); @@ -264,16 +270,16 @@ fn calvin_static_panic_rollback_not_visible() { /// After clearing the fail point, Calvin transactions must commit /// successfully. This confirms there is no state corruption from the earlier -/// panicking flush. +/// panicking install. #[cfg(feature = "failpoints")] #[test] fn calvin_static_normal_operation_resumes_after_panic() { let (mut core, mut tx, mut rx, _dir) = make_core(); - // Trigger a panicking flush. + // Trigger a panicking install. { let _guard = FailGuard::install(INSTALL_FAIL_POINT, FailAction::Panic); - let resp = stage_then_flush( + let resp = stage_then_install( &mut core, &mut tx, &mut rx, @@ -286,14 +292,14 @@ fn calvin_static_normal_operation_resumes_after_panic() { assert_eq!( resp.status, Status::Error, - "a panicking flush must return Error; got {:?}", + "a panicking install must return Error; got {:?}", resp.status ); // Guard drops here, clearing the fail point. } // A normal Calvin transaction after the fail point is cleared. - let success_resp = stage_then_flush( + let success_resp = stage_then_install( &mut core, &mut tx, &mut rx, @@ -303,7 +309,7 @@ fn calvin_static_normal_operation_resumes_after_panic() { assert_eq!( success_resp.status, Status::Ok, - "a Calvin flush after the fail point is cleared must succeed; got {:?}", + "a Calvin install after the fail point is cleared must succeed; got {:?}", success_resp.error_code ); @@ -328,13 +334,13 @@ fn calvin_static_normal_operation_resumes_after_panic() { // ── Test 4: WAL replay correctness — fresh CoreLoop sees only committed data ── -/// After a panicking Calvin flush, create a fresh `CoreLoop` at the same data -/// directory. The fresh core must see only data that was committed before the -/// panicking flush — the rolled-back writes must not appear after replay. +/// After a panicking Calvin install, create a fresh `CoreLoop` at the same +/// data directory. The fresh core must see only data that was committed before +/// the panicking install — the rolled-back writes must not appear after replay. /// /// The install rolled its writes back before the response returned, so a /// fresh core opened over the same directory holds none of them. The redo -/// record itself lives in the WAL the scheduler appends, which this +/// record itself lives in the WAL the data group's apply appends, which this /// core-level test does not write. #[cfg(feature = "failpoints")] #[test] @@ -364,36 +370,17 @@ fn calvin_static_replay_sees_only_committed_data() { (core, req_tx, resp_rx) }; - // Commit a reference write before the panicking flush. + // Commit a reference write before the panicking install. { - use nodedb::bridge::dispatch::BridgeRequest; - use nodedb::bridge::envelope::{Priority, Request}; + use nodedb::bridge::envelope::Request; use nodedb_physical::physical_plan::PhysicalPlan; - use std::time::{Duration, Instant}; let make_req = |plan: PhysicalPlan| Request { request_id: nodedb::types::RequestId::new(42), - tenant_id: nodedb::types::TenantId::new(1), - vshard_id: nodedb::types::VShardId::new(0), - database_id: nodedb::types::DatabaseId::DEFAULT, - plan, - deadline: Instant::now() + Duration::from_secs(5), - priority: Priority::Normal, - trace_id: nodedb_types::TraceId::ZERO, - consistency: nodedb::types::ReadConsistency::Strong, - idempotency_key: None, - event_source: nodedb::event::EventSource::User, - user_roles: Vec::new(), - user_id: None, - statement_digest: None, - txn_id: None, - wal_lsn: None, - resolved_now_ms: None, - commit_hlc: None, - admission: nodedb::bridge::envelope::Admission::Admitted, + ..make_request(plan) }; - // Commit a value before the panicking flush. + // Commit a value before the panicking install. tx.try_push(BridgeRequest::unfloored(make_req(kv_put_in( "replay_coll", b"pre_commit", @@ -410,9 +397,9 @@ fn calvin_static_replay_sees_only_committed_data() { ); // Panicking install — writes must not persist. Stage and resolve - // write nothing; the panic fires while the flush installs. + // write nothing; the panic fires while the install writes. let _guard = FailGuard::install(INSTALL_FAIL_POINT, FailAction::Panic); - let panic_resp = stage_then_flush( + let panic_resp = stage_then_install( &mut core, &mut tx, &mut rx, @@ -447,33 +434,15 @@ fn calvin_static_replay_sees_only_committed_data() { (core, req_tx, resp_rx) }; - use nodedb::bridge::dispatch::BridgeRequest; - use nodedb::bridge::envelope::{Priority, Request}; + use nodedb::bridge::envelope::Request; use nodedb_physical::physical_plan::PhysicalPlan; - use std::time::{Duration, Instant}; let make_req2 = |plan: PhysicalPlan| Request { request_id: nodedb::types::RequestId::new(43), - tenant_id: nodedb::types::TenantId::new(1), - vshard_id: nodedb::types::VShardId::new(0), - database_id: nodedb::types::DatabaseId::DEFAULT, - plan, - deadline: Instant::now() + Duration::from_secs(5), - priority: Priority::Normal, - trace_id: nodedb_types::TraceId::ZERO, - consistency: nodedb::types::ReadConsistency::Strong, - idempotency_key: None, - event_source: nodedb::event::EventSource::User, - user_roles: Vec::new(), - user_id: None, - statement_digest: None, - txn_id: None, - wal_lsn: None, - resolved_now_ms: None, - commit_hlc: None, admission: nodedb::bridge::envelope::Admission::Exempt( nodedb::bridge::envelope::ExemptReason::Read, ), + ..make_request(plan) }; // Note: this test does NOT assert that the pre-panic committed value is @@ -482,7 +451,7 @@ fn calvin_static_replay_sees_only_committed_data() { // on a single-CoreLoop reopen. WAL-driven KV state recovery is a property // of the cluster apply path (replicated entries → applier → engine), not // of CoreLoop::open. The meaningful invariant exercised below is that the - // rolled-back writes from the panicking flush do NOT appear, which holds + // rolled-back writes from the panicking install do NOT appear, which holds // trivially under empty-replay state and confirms the panic-rollback path // never let the bad writes reach durable storage. diff --git a/nodedb/tests/inproc/cases/calvin_executor_static_multishard.rs b/nodedb/tests/inproc/cases/calvin_executor_static_multishard.rs index 8116b1e73..f73ab597b 100644 --- a/nodedb/tests/inproc/cases/calvin_executor_static_multishard.rs +++ b/nodedb/tests/inproc/cases/calvin_executor_static_multishard.rs @@ -21,7 +21,6 @@ fn calvin_execute_static_round_trip_msgpack() { tenant_id: TenantId::new(1), plans: vec![], epoch_system_ms: 0, - is_group_leader: true, versioned_reads: vec![], body_plans: vec![], }; @@ -59,7 +58,6 @@ fn calvin_execute_static_and_active_are_distinct_variants() { tenant_id: TenantId::new(1), plans: vec![], epoch_system_ms: 0, - is_group_leader: true, versioned_reads: vec![], body_plans: vec![], }; diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply.rs deleted file mode 100644 index 5eee3f8bc..000000000 --- a/nodedb/tests/inproc/cases/calvin_two_phase_apply.rs +++ /dev/null @@ -1,908 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Staged Calvin apply on the Data Plane: `MetaOp::CalvinExecuteStatic` -//! VALIDATES + STAGES a transaction's write plans into the commit-pending -//! buffer WITHOUT mutating base, returning the local commit vote on -//! `stage_vote`. `MetaOp::CalvinResolve` then resolves the staged plans -//! into the transaction's redo record, and `MetaOp::CalvinFlush` installs it -//! at its LSN (making the write visible), or `MetaOp::CalvinDrop` discards -//! the staged state (leaving base unchanged). -//! -//! These drive a `CoreLoop` directly through the SPSC ring so the atomicity -//! seam is observed without any scheduler timing: nothing a stage writes is -//! visible until the flush, and a drop never makes it visible. - -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use nodedb::bridge::dispatch::{BridgeRequest, BridgeResponse}; -use nodedb::bridge::envelope::{Priority, Request, Response, StageVote, Status}; -use nodedb::data::executor::core_loop::CoreLoop; -use nodedb::types::*; -use nodedb_bridge::buffer::{Consumer, Producer, RingBuffer}; -use nodedb_physical::physical_plan::{DocumentOp, KvOp, MetaOp, PhysicalPlan}; -use nodedb_types::QualifiedCollection; -use nodedb_types::Surrogate; -use nodedb_types::calvin::{EngineTag, ReadKeyIdent, VersionedReadEntry}; - -fn make_core() -> ( - CoreLoop, - Producer, - Consumer, - tempfile::TempDir, -) { - let dir = tempfile::tempdir().unwrap(); - let (req_tx, req_rx) = RingBuffer::channel::(64); - let (resp_tx, resp_rx) = RingBuffer::channel::(64); - let core = CoreLoop::open( - 0, - req_rx, - resp_tx, - dir.path(), - Arc::new(nodedb_types::OrdinalClock::new()), - nodedb::data::executor::core_loop::test_governor(), - ) - .unwrap(); - (core, req_tx, resp_rx, dir) -} - -/// Build a request for `plan` on `vshard`, carrying an optional committed WAL -/// LSN (present on the seed write so its version is recorded). -fn make_request(plan: PhysicalPlan, vshard: u32, wal_lsn: Option) -> Request { - Request { - request_id: RequestId::new(1), - tenant_id: TenantId::new(1), - vshard_id: VShardId::new(vshard), - database_id: DatabaseId::DEFAULT, - plan, - deadline: Instant::now() + Duration::from_secs(5), - priority: Priority::Normal, - trace_id: nodedb_types::TraceId::ZERO, - consistency: ReadConsistency::Strong, - idempotency_key: None, - event_source: nodedb::event::EventSource::RaftFollower, - user_roles: Vec::new(), - user_id: None, - statement_digest: None, - txn_id: None, - wal_lsn, - resolved_now_ms: None, - commit_hlc: None, - admission: nodedb::bridge::envelope::Admission::Admitted, - } -} - -fn send( - core: &mut CoreLoop, - tx: &mut Producer, - rx: &mut Consumer, - plan: PhysicalPlan, - vshard: u32, - wal_lsn: Option, -) -> Response { - tx.try_push(BridgeRequest::unfloored(make_request( - plan, vshard, wal_lsn, - ))) - .unwrap(); - core.tick(); - rx.try_pop().unwrap().inner -} - -/// Resolve the transaction staged at `(epoch, 0)` on `vshard` and build the -/// flush that installs its redo record. -fn resolved_flush( - core: &mut CoreLoop, - tx: &mut Producer, - rx: &mut Consumer, - epoch: u64, - vshard: u32, -) -> PhysicalPlan { - let resolved = send( - core, - tx, - rx, - PhysicalPlan::Meta(MetaOp::CalvinResolve { epoch, position: 0 }), - vshard, - None, - ); - assert_eq!( - resolved.status, - Status::Ok, - "resolve must succeed: {resolved:?}" - ); - PhysicalPlan::Meta(MetaOp::CalvinFlush { - epoch, - position: 0, - redo: resolved_redo(&resolved), - collections: Vec::new(), - sum_targets: Vec::new(), - }) -} - -/// The redo record a `CalvinResolve` answered with. -fn resolved_redo(resolved: &Response) -> Vec { - let answer: nodedb_physical::physical_plan::CalvinResolved = - zerompk::from_msgpack(resolved.payload.as_bytes()).expect("decode resolved answer"); - answer.redo -} - -/// A write committed through the Calvin path to seed a write version. -struct CalvinSeed<'a> { - epoch: u64, - vshard: u32, - /// The collection `plans` write; the install records its floor at `lsn`. - collection: &'a str, - plans: Vec, - lsn: u64, -} - -/// Commit `seed.plans` as the Calvin transaction at `(seed.epoch, 0)` on -/// `seed.vshard` and install its redo record at `seed.lsn`: the path a -/// committed multi-shard write takes. The install records each written key -/// and the collection floor at that LSN. -fn commit_calvin( - core: &mut CoreLoop, - tx: &mut Producer, - rx: &mut Consumer, - seed: CalvinSeed<'_>, -) -> Response { - let staged = send( - core, - tx, - rx, - stage_static(seed.epoch, 0, seed.plans, Vec::new()), - seed.vshard, - None, - ); - assert_eq!( - staged.status, - Status::Ok, - "seed stage must succeed: {staged:?}" - ); - let resolved = send( - core, - tx, - rx, - PhysicalPlan::Meta(MetaOp::CalvinResolve { - epoch: seed.epoch, - position: 0, - }), - seed.vshard, - None, - ); - assert_eq!( - resolved.status, - Status::Ok, - "seed resolve must succeed: {resolved:?}" - ); - send( - core, - tx, - rx, - PhysicalPlan::Meta(MetaOp::CalvinFlush { - epoch: seed.epoch, - position: 0, - redo: resolved_redo(&resolved), - collections: vec![seed.collection.to_string()], - sum_targets: Vec::new(), - }), - seed.vshard, - Some(Lsn::new(seed.lsn)), - ) -} - -fn kv_put(coll: &str, key: &[u8], value: &[u8]) -> PhysicalPlan { - PhysicalPlan::Kv(KvOp::Put { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), - key: key.to_vec(), - value: value.to_vec(), - ttl_ms: 0, - surrogate: nodedb_test_support::kv_rows::kv_row_surrogate(key), - returning: None, - rls_filters: Vec::new(), - provenance: None, - }) -} - -fn kv_get(coll: &str, key: &[u8]) -> PhysicalPlan { - PhysicalPlan::Kv(KvOp::Get { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), - key: key.to_vec(), - rls_filters: Vec::new(), - surrogate_ceiling: None, - }) -} - -/// A minimal msgpack-encoded document body, `{"a": "1"}`. -fn doc_value() -> Vec { - let mut obj = std::collections::HashMap::new(); - obj.insert("a".to_string(), nodedb_types::Value::String("1".into())); - zerompk::to_msgpack_vec(&nodedb_types::Value::Object(obj)).unwrap() -} - -/// A minimal document `PointInsert` plan, used to record a write-version at a -/// specific surrogate (the document engine's per-key identity). -fn doc_insert(coll: &str, document_id: &str, surrogate: u32) -> PhysicalPlan { - PhysicalPlan::Document(DocumentOp::PointInsert { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), - document_id: document_id.to_string(), - value: doc_value(), - if_absent: false, - surrogate: Surrogate::new(surrogate), - returning: None, - rls_filters: Vec::new(), - resolved_sum_targets: Vec::new(), - deferred_sum_targets: Vec::new(), - }) -} - -fn stage_static( - epoch: u64, - position: u32, - plans: Vec, - versioned_reads: Vec, -) -> PhysicalPlan { - PhysicalPlan::Meta(MetaOp::CalvinExecuteStatic { - epoch, - position, - tenant_id: TenantId::new(1), - plans, - epoch_system_ms: 0, - is_group_leader: true, - versioned_reads, - body_plans: Vec::new(), - }) -} - -/// A valid (empty read-set) staged write is NOT visible until the flush, and -/// the flush makes it visible — the stage/flush atomicity seam. -#[test] -fn flush_makes_staged_calvin_write_visible() { - let (mut core, mut tx, mut rx, _dir) = make_core(); - - // Stage a write with an empty read-set → vote is valid (commit). - let staged = send( - &mut core, - &mut tx, - &mut rx, - stage_static(6, 0, vec![kv_put("flushcoll", b"fk", b"fv")], Vec::new()), - 0, - None, - ); - assert_eq!(staged.status, Status::Ok, "stage must succeed"); - assert_eq!( - staged.stage_vote, - Some(StageVote::Commit), - "empty read-set is vacuously current → commit vote" - ); - - // Not yet applied: the staged write is invisible before the flush. - let before = send( - &mut core, - &mut tx, - &mut rx, - kv_get("flushcoll", b"fk"), - 0, - None, - ); - assert!( - before.payload.is_empty() || before.status == Status::Error, - "staged write must NOT be visible before flush; got {before:?}" - ); - - // The flush installs the resolved redo record to base. - let flush_plan = resolved_flush(&mut core, &mut tx, &mut rx, 6, 0); - let flush = send( - &mut core, - &mut tx, - &mut rx, - flush_plan, - 0, - Some(Lsn::new(60)), - ); - assert_eq!(flush.status, Status::Ok, "flush must succeed: {flush:?}"); - - // Now visible. - let after = send( - &mut core, - &mut tx, - &mut rx, - kv_get("flushcoll", b"fk"), - 0, - None, - ); - assert_eq!( - after.status, - Status::Ok, - "read after flush must succeed: {after:?}" - ); - assert!( - !after.payload.is_empty(), - "flushed write must be visible after flush" - ); -} - -/// An invalid vote (a stale versioned read against a newer local write) STAGES -/// but never applies; the drop discards it and base stays unchanged. -#[test] -fn drop_discards_invalid_staged_calvin_write() { - let (mut core, mut tx, mut rx, _dir) = make_core(); - - // Seed a committed write to `dropcoll` at LSN 100 so its collection write - // version floor is 100. The seed carries a WAL LSN so the version records. - let seed = commit_calvin( - &mut core, - &mut tx, - &mut rx, - CalvinSeed { - epoch: 1, - vshard: 0, - collection: "dropcoll", - plans: vec![kv_put("dropcoll", b"seed", b"v")], - lsn: 100, - }, - ); - assert_eq!(seed.status, Status::Ok, "seed write must commit: {seed:?}"); - - // The read entry's collection must home to the staged request's vShard for - // the read-set check to consider it. - let read_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "dropcoll") - .vshard() - .as_u32(); - - // A read of `dropcoll` observed at LSN 50 — stale against the seed's write - // at LSN 100 → the read-set is no longer current → abort vote. - let stale_read = VersionedReadEntry { - engine: EngineTag::Kv, - collection: "dropcoll".to_string(), - key: ReadKeyIdent::Predicate, - read_lsn: Lsn::new(50), - home_vshard: None, - served_by: 0, - }; - - let staged = send( - &mut core, - &mut tx, - &mut rx, - stage_static( - 7, - 0, - vec![kv_put("targetcoll", b"tk", b"tv")], - vec![stale_read], - ), - read_vshard, - None, - ); - assert_eq!( - staged.status, - Status::Ok, - "stage must succeed even on abort vote" - ); - assert_eq!( - staged.stage_vote, - Some(StageVote::SerializationConflict), - "stale read-set must produce an abort vote" - ); - - // Staged, not applied: the target write is invisible. - let before = send( - &mut core, - &mut tx, - &mut rx, - kv_get("targetcoll", b"tk"), - 0, - None, - ); - assert!( - before.payload.is_empty() || before.status == Status::Error, - "aborted staged write must NOT be visible; got {before:?}" - ); - - // Drop discards the staged plans and fires nothing. The drop must target - // the SAME vShard the stage keyed under (as production dispatches it), so it - // actually pops this participant's staged slice. - let dropped = send( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Meta(MetaOp::CalvinDrop { - epoch: 7, - position: 0, - }), - read_vshard, - None, - ); - assert_eq!(dropped.status, Status::Ok, "drop must succeed: {dropped:?}"); - - // Still invisible after the drop — base was never mutated. - let after = send( - &mut core, - &mut tx, - &mut rx, - kv_get("targetcoll", b"tk"), - 0, - None, - ); - assert!( - after.payload.is_empty() || after.status == Status::Error, - "dropped write must never be visible; got {after:?}" - ); -} - -/// A `Point` read (not just a `Predicate` read) at or after the recorded write -/// LSN is current → commit vote, and the flush applies the staged write. -#[test] -fn point_read_at_write_lsn_commits_and_flush_applies() { - let (mut core, mut tx, mut rx, _dir) = make_core(); - - // Seed a committed write to key `pk` in `pointcoll` at LSN 10. - let seed = commit_calvin( - &mut core, - &mut tx, - &mut rx, - CalvinSeed { - epoch: 1, - vshard: 0, - collection: "pointcoll", - plans: vec![kv_put("pointcoll", b"pk", b"v1")], - lsn: 10, - }, - ); - assert_eq!(seed.status, Status::Ok, "seed write must commit: {seed:?}"); - - let point_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "pointcoll") - .vshard() - .as_u32(); - - // A Point read of the exact same key observed at LSN 10 (== the write) is - // still current: no write happened AFTER the read. - let current_read = VersionedReadEntry { - engine: EngineTag::Kv, - collection: "pointcoll".to_string(), - key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"pk".as_slice()))), - read_lsn: Lsn::new(10), - home_vshard: None, - served_by: 0, - }; - - let staged = send( - &mut core, - &mut tx, - &mut rx, - stage_static( - 8, - 0, - vec![kv_put("committarget", b"ck", b"cv")], - vec![current_read], - ), - point_vshard, - None, - ); - assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); - assert_eq!( - staged.stage_vote, - Some(StageVote::Commit), - "a read at or after the last write LSN must be current -> commit vote" - ); - - let flush_plan = resolved_flush(&mut core, &mut tx, &mut rx, 8, point_vshard); - let flush = send( - &mut core, - &mut tx, - &mut rx, - flush_plan, - point_vshard, - Some(Lsn::new(80)), - ); - assert_eq!(flush.status, Status::Ok, "flush must succeed: {flush:?}"); - - let after = send( - &mut core, - &mut tx, - &mut rx, - kv_get("committarget", b"ck"), - 0, - None, - ); - assert_eq!(after.status, Status::Ok, "read after flush must succeed"); - assert!( - !after.payload.is_empty(), - "flushed write must be visible after flush" - ); -} - -/// A `Point` read of a key STALE against a later write to that same key -/// (read_lsn=5 vs. a committed write at LSN=10) aborts the stage vote, and the -/// drop discards the staged write with no base mutation — mirrors -/// `drop_discards_invalid_staged_calvin_write` but for a Point key (not a -/// collection-scoped Predicate). -#[test] -fn stale_point_read_of_kv_key_aborts_stage_and_drop_discards() { - let (mut core, mut tx, mut rx, _dir) = make_core(); - - // Seed a committed write to key `pk` in `stalecoll` at LSN 10. - let seed = commit_calvin( - &mut core, - &mut tx, - &mut rx, - CalvinSeed { - epoch: 1, - vshard: 0, - collection: "stalecoll", - plans: vec![kv_put("stalecoll", b"pk", b"v1")], - lsn: 10, - }, - ); - assert_eq!(seed.status, Status::Ok, "seed write must commit: {seed:?}"); - - let stale_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "stalecoll") - .vshard() - .as_u32(); - - // A Point read of the exact same key observed at LSN 5 — stale against the - // write at LSN 10 → the read-set is no longer current → abort vote. - let stale_read = VersionedReadEntry { - engine: EngineTag::Kv, - collection: "stalecoll".to_string(), - key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"pk".as_slice()))), - read_lsn: Lsn::new(5), - home_vshard: None, - served_by: 0, - }; - - let staged = send( - &mut core, - &mut tx, - &mut rx, - stage_static( - 9, - 0, - vec![kv_put("aborttarget", b"ak", b"av")], - vec![stale_read], - ), - stale_vshard, - None, - ); - assert_eq!( - staged.status, - Status::Ok, - "stage must succeed even on abort vote" - ); - assert_eq!( - staged.stage_vote, - Some(StageVote::SerializationConflict), - "a stale Point read (write after read_lsn) must produce an abort vote" - ); - - let before = send( - &mut core, - &mut tx, - &mut rx, - kv_get("aborttarget", b"ak"), - 0, - None, - ); - assert!( - before.payload.is_empty() || before.status == Status::Error, - "aborted staged write must NOT be visible; got {before:?}" - ); - - let dropped = send( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Meta(MetaOp::CalvinDrop { - epoch: 9, - position: 0, - }), - stale_vshard, - None, - ); - assert_eq!(dropped.status, Status::Ok, "drop must succeed: {dropped:?}"); - - let after = send( - &mut core, - &mut tx, - &mut rx, - kv_get("aborttarget", b"ak"), - 0, - None, - ); - assert!( - after.payload.is_empty() || after.status == Status::Error, - "dropped write must never be visible; got {after:?}" - ); -} - -/// Phantom-insert proof for the KV engine: a Point read observed a key as -/// ABSENT at LSN 5; a concurrent INSERT of that EXACT key commits at LSN 8. -/// The KV read-key identity is the raw key bytes — the same identity the -/// insert writes under — so the per-key write-version check catches the -/// phantom: validating the stale absent-read against the now-present key -/// aborts the stage. -#[test] -fn absent_kv_key_phantom_insert_causes_abort() { - let (mut core, mut tx, mut rx, _dir) = make_core(); - - let phantom_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "phantomkv") - .vshard() - .as_u32(); - - // The key was absent when read at (the then-current watermark) LSN 5 — no - // write is seeded yet. - let absent_read = VersionedReadEntry { - engine: EngineTag::Kv, - collection: "phantomkv".to_string(), - key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"newkey".as_slice()))), - read_lsn: Lsn::new(5), - home_vshard: None, - served_by: 0, - }; - - // Concurrently, the exact same key is inserted and commits at LSN 8. - let insert = commit_calvin( - &mut core, - &mut tx, - &mut rx, - CalvinSeed { - epoch: 1, - vshard: phantom_vshard, - collection: "phantomkv", - plans: vec![PhysicalPlan::Kv(KvOp::Insert { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "phantomkv"), - key: b"newkey".to_vec(), - value: b"v".to_vec(), - ttl_ms: 0, - surrogate: nodedb_test_support::kv_rows::kv_row_surrogate(b"newkey".as_ref()), - returning: None, - rls_filters: Vec::new(), - })], - lsn: 8, - }, - ); - assert_eq!( - insert.status, - Status::Ok, - "phantom insert must commit: {insert:?}" - ); - - // Validating the (now stale) absent-read against the current write index - // must abort: the insert at LSN 8 is AFTER the read at LSN 5. - let staged = send( - &mut core, - &mut tx, - &mut rx, - stage_static( - 10, - 0, - vec![kv_put("phantomtarget", b"tk", b"tv")], - vec![absent_read], - ), - phantom_vshard, - None, - ); - assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); - assert_eq!( - staged.stage_vote, - Some(StageVote::SerializationConflict), - "a phantom insert into a key observed absent must abort the stage vote" - ); -} - -/// Absent-DOCUMENT phantom safety: a document `PointGet` on an absent document -/// cannot record `Point(KeyRepr::Surrogate(s))`, because the placeholder -/// surrogate `s` a miss carries is allocated from a monotonic counter unrelated -/// to `document_id` — it never coincides with the freshly-minted surrogate a -/// subsequent INSERT of that `document_id` receives, so a per-key OCC check -/// would never catch the phantom. The capture layer therefore degrades an -/// absent document read to `Predicate` (collection floor). A concurrent INSERT -/// into that collection advances `coll_write_lsn` past the read_lsn, so -/// `WriteVersionIndex::read_is_valid` (predicate branch) judges the stale read -/// invalid and the stage VOTES ABORT — collection-granular phantom safety. -#[test] -fn absent_document_phantom_insert_is_caught() { - let (mut core, mut tx, mut rx, _dir) = make_core(); - - let doc_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "phantomdocs") - .vshard() - .as_u32(); - - // The document was absent when read: capture degraded the miss to a - // collection-scoped predicate on "phantomdocs" at read_lsn 5. - let absent_doc_read = VersionedReadEntry { - engine: EngineTag::Document, - collection: "phantomdocs".to_string(), - key: ReadKeyIdent::Predicate, - read_lsn: Lsn::new(5), - home_vshard: None, - served_by: 0, - }; - - // Concurrently, a document with the same document_id the read targeted is - // actually inserted -- allocated a freshly-minted surrogate (42), committing - // at LSN 8. Its collection floor advance (phantomdocs -> 8) is what the - // predicate read validates against. - const NEWLY_ALLOCATED_SURROGATE: u32 = 42; - let insert = commit_calvin( - &mut core, - &mut tx, - &mut rx, - CalvinSeed { - epoch: 1, - vshard: doc_vshard, - collection: "phantomdocs", - plans: vec![doc_insert( - "phantomdocs", - "the-doc-id", - NEWLY_ALLOCATED_SURROGATE, - )], - lsn: 8, - }, - ); - assert_eq!( - insert.status, - Status::Ok, - "phantom insert must commit: {insert:?}" - ); - - // Validate the stale absent-read: the predicate check sees the phantomdocs - // collection floor (LSN 8) is AFTER the read (LSN 5), so the read is no - // longer current and the stage must abort. - let staged = send( - &mut core, - &mut tx, - &mut rx, - stage_static( - 11, - 0, - vec![kv_put("docphantomtarget", b"tk", b"tv")], - vec![absent_doc_read], - ), - doc_vshard, - None, - ); - assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); - assert_eq!( - staged.stage_vote, - Some(StageVote::SerializationConflict), - "a phantom insert into a collection observed absent must abort the stage \ - vote via the collection floor" - ); -} - -/// No-over-abort companion to `absent_document_phantom_insert_is_caught`: the -/// collection-floor degrade must NOT abort every absent-read transaction. An -/// absent document read (predicate on "phantomdocs" at LSN 5) stays valid when -/// no insert lands in THAT collection — a concurrent insert into an UNRELATED -/// collection advances only its own floor, leaving phantomdocs at zero, so the -/// stale read is still current and the stage VOTES COMMIT. -#[test] -fn absent_document_read_without_matching_insert_still_commits() { - let (mut core, mut tx, mut rx, _dir) = make_core(); - - let doc_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "phantomdocs") - .vshard() - .as_u32(); - - let absent_doc_read = VersionedReadEntry { - engine: EngineTag::Document, - collection: "phantomdocs".to_string(), - key: ReadKeyIdent::Predicate, - read_lsn: Lsn::new(5), - home_vshard: None, - served_by: 0, - }; - - // A concurrent insert into a DIFFERENT collection commits at LSN 8. It - // advances only "othercoll"'s floor; phantomdocs is untouched. - const UNRELATED_SURROGATE: u32 = 42; - let insert = commit_calvin( - &mut core, - &mut tx, - &mut rx, - CalvinSeed { - epoch: 1, - vshard: doc_vshard, - collection: "othercoll", - plans: vec![doc_insert("othercoll", "other-id", UNRELATED_SURROGATE)], - lsn: 8, - }, - ); - assert_eq!( - insert.status, - Status::Ok, - "unrelated insert must commit: {insert:?}" - ); - - // The predicate read on phantomdocs is still current: its collection floor - // never advanced past read_lsn 5, so the stage must commit. - let staged = send( - &mut core, - &mut tx, - &mut rx, - stage_static( - 11, - 0, - vec![kv_put("docphantomtarget", b"tk", b"tv")], - vec![absent_doc_read], - ), - doc_vshard, - None, - ); - assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); - assert_eq!( - staged.stage_vote, - Some(StageVote::Commit), - "an absent-document read must NOT over-abort when no insert lands in its \ - collection" - ); -} - -/// Push one prebuilt request through the ring and return its response. -fn send_request( - core: &mut CoreLoop, - tx: &mut Producer, - rx: &mut Consumer, - request: Request, -) -> Response { - tx.try_push(BridgeRequest::unfloored(request)).unwrap(); - core.tick(); - rx.try_pop().unwrap().inner -} - -/// Calvin sub-operations are already ordered: every replica must run them. -/// A stage and a flush whose envelope deadline has already passed still -/// execute, and the write becomes visible. They never answer -/// `DeadlineExceeded`. -#[test] -fn already_ordered_stage_and_flush_run_past_their_deadline() { - let (mut core, mut tx, mut rx, _dir) = make_core(); - let already_ordered = |plan: PhysicalPlan| Request { - deadline: Instant::now() - Duration::from_secs(1), - admission: nodedb::bridge::envelope::Admission::Exempt( - nodedb::bridge::envelope::ExemptReason::AlreadyOrdered, - ), - ..make_request(plan, 0, None) - }; - - let staged = send_request( - &mut core, - &mut tx, - &mut rx, - already_ordered(stage_static( - 9, - 0, - vec![kv_put("latecoll", b"lk", b"lv")], - Vec::new(), - )), - ); - assert_eq!(staged.status, Status::Ok, "late stage must run: {staged:?}"); - assert_eq!(staged.stage_vote, Some(StageVote::Commit)); - - let flush_plan = resolved_flush(&mut core, &mut tx, &mut rx, 9, 0); - let flush = send_request( - &mut core, - &mut tx, - &mut rx, - Request { - wal_lsn: Some(Lsn::new(90)), - ..already_ordered(flush_plan) - }, - ); - assert_eq!(flush.status, Status::Ok, "late flush must run: {flush:?}"); - - let after = send( - &mut core, - &mut tx, - &mut rx, - kv_get("latecoll", b"lk"), - 0, - None, - ); - assert_eq!(after.status, Status::Ok, "read after flush: {after:?}"); - assert!( - !after.payload.is_empty(), - "the late flush must make the staged write visible" - ); -} diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply/install.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply/install.rs new file mode 100644 index 000000000..7e92a4772 --- /dev/null +++ b/nodedb/tests/inproc/cases/calvin_two_phase_apply/install.rs @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A staged write becomes visible only at its stamped install. + +use std::time::{Duration, Instant}; + +use nodedb::bridge::envelope::{Request, StageVote, Status}; +use nodedb::types::*; +use nodedb_physical::physical_plan::PhysicalPlan; + +use super::support::*; + +/// A valid (empty read-set) staged write is NOT visible until the stamped +/// install, and the install makes it visible: the stage/install atomicity seam. +#[test] +fn the_stamped_install_makes_a_staged_calvin_write_visible() { + let (mut core, mut tx, mut rx, _dir) = make_core(); + + // Stage a write with an empty read-set → vote is valid (commit). + let staged = send( + &mut core, + &mut tx, + &mut rx, + stage_static(6, 0, vec![kv_put("flushcoll", b"fk", b"fv")], Vec::new()), + 0, + None, + ); + assert_eq!(staged.status, Status::Ok, "stage must succeed"); + assert_eq!( + staged.stage_vote, + Some(StageVote::Commit), + "empty read-set is vacuously current → commit vote" + ); + + // Not yet applied: the staged write is invisible before the install. + let before = send( + &mut core, + &mut tx, + &mut rx, + kv_get("flushcoll", b"fk"), + 0, + None, + ); + assert!( + before.payload.is_empty() || before.status == Status::Error, + "staged write must NOT be visible before the install; got {before:?}" + ); + + // The stamped install writes the resolved redo record to base. + let install_plan = resolved_install(&mut core, &mut tx, &mut rx, 6, 0); + let installed = send( + &mut core, + &mut tx, + &mut rx, + install_plan, + 0, + Some(Lsn::new(60)), + ); + assert_eq!( + installed.status, + Status::Ok, + "install must succeed: {installed:?}" + ); + + // Now visible. + let after = send( + &mut core, + &mut tx, + &mut rx, + kv_get("flushcoll", b"fk"), + 0, + None, + ); + assert_eq!( + after.status, + Status::Ok, + "read after the install must succeed: {after:?}" + ); + assert!( + !after.payload.is_empty(), + "the installed write must be visible after the install" + ); +} + +/// Calvin sub-operations are already ordered: every replica must run them. +/// A stage and a stamped install whose envelope deadline has already passed +/// still execute, and the write becomes visible. They never answer +/// `DeadlineExceeded`. +#[test] +fn already_ordered_stage_and_install_run_past_their_deadline() { + let (mut core, mut tx, mut rx, _dir) = make_core(); + let already_ordered = |plan: PhysicalPlan| Request { + deadline: Instant::now() - Duration::from_secs(1), + admission: nodedb::bridge::envelope::Admission::Exempt( + nodedb::bridge::envelope::ExemptReason::AlreadyOrdered, + ), + ..make_request(plan, 0, None) + }; + + let staged = send_request( + &mut core, + &mut tx, + &mut rx, + already_ordered(stage_static( + 9, + 0, + vec![kv_put("latecoll", b"lk", b"lv")], + Vec::new(), + )), + ); + assert_eq!(staged.status, Status::Ok, "late stage must run: {staged:?}"); + assert_eq!(staged.stage_vote, Some(StageVote::Commit)); + + let install_plan = resolved_install(&mut core, &mut tx, &mut rx, 9, 0); + let installed = send_request( + &mut core, + &mut tx, + &mut rx, + Request { + wal_lsn: Some(Lsn::new(90)), + ..already_ordered(install_plan) + }, + ); + assert_eq!( + installed.status, + Status::Ok, + "late install must run: {installed:?}" + ); + + let after = send( + &mut core, + &mut tx, + &mut rx, + kv_get("latecoll", b"lk"), + 0, + None, + ); + assert_eq!( + after.status, + Status::Ok, + "read after the install: {after:?}" + ); + assert!( + !after.payload.is_empty(), + "the late install must make the staged write visible" + ); +} diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply/mod.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply/mod.rs new file mode 100644 index 000000000..aa222a58d --- /dev/null +++ b/nodedb/tests/inproc/cases/calvin_two_phase_apply/mod.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Staged Calvin apply on the Data Plane: `MetaOp::CalvinExecuteStatic` +//! VALIDATES + STAGES a transaction's write plans into the commit-pending +//! buffer WITHOUT mutating base, returning the local commit vote on +//! `stage_vote`. `MetaOp::CalvinResolve` then resolves the staged plans +//! into the transaction's redo record. The data group's apply of the slice's +//! stamped redo entry (`MetaOp::ApplyTransactionRedo` with a `CalvinInstall`) +//! installs it at its LSN (making the write visible), or `MetaOp::CalvinDrop` +//! discards the staged state (leaving base unchanged). +//! +//! These drive a `CoreLoop` directly through the SPSC ring so the atomicity +//! seam is observed without any scheduler timing: nothing a stage writes is +//! visible until the install, and a drop never makes it visible. + +mod install; +mod phantom; +mod read_set; +mod support; diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply/phantom.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply/phantom.rs new file mode 100644 index 000000000..8f39a4f74 --- /dev/null +++ b/nodedb/tests/inproc/cases/calvin_two_phase_apply/phantom.rs @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A phantom insert into a key or collection a slice read as absent +//! aborts the stage vote. An unrelated insert does not. + +use nodedb::bridge::envelope::{StageVote, Status}; +use nodedb::types::*; +use nodedb_physical::physical_plan::{KvOp, PhysicalPlan}; +use nodedb_types::QualifiedCollection; +use nodedb_types::calvin::{EngineTag, ReadKeyIdent, VersionedReadEntry}; + +use super::support::*; + +/// Phantom-insert proof for the KV engine: a Point read observed a key as +/// ABSENT at LSN 5; a concurrent INSERT of that EXACT key commits at LSN 8. +/// The KV read-key identity is the raw key bytes — the same identity the +/// insert writes under — so the per-key write-version check catches the +/// phantom: validating the stale absent-read against the now-present key +/// aborts the stage. +#[test] +fn absent_kv_key_phantom_insert_causes_abort() { + let (mut core, mut tx, mut rx, _dir) = make_core(); + + let phantom_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "phantomkv") + .vshard() + .as_u32(); + + // The key was absent when read at (the then-current watermark) LSN 5 — no + // write is seeded yet. + let absent_read = VersionedReadEntry { + engine: EngineTag::Kv, + collection: "phantomkv".to_string(), + key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"newkey".as_slice()))), + read_lsn: Lsn::new(5), + home_vshard: None, + served_by: 0, + }; + + // Concurrently, the exact same key is inserted and commits at LSN 8. + let insert = commit_calvin( + &mut core, + &mut tx, + &mut rx, + CalvinSeed { + epoch: 1, + vshard: phantom_vshard, + collection: "phantomkv", + plans: vec![PhysicalPlan::Kv(KvOp::Insert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "phantomkv"), + key: b"newkey".to_vec(), + value: b"v".to_vec(), + ttl_ms: 0, + surrogate: nodedb_test_support::kv_rows::kv_row_surrogate(b"newkey".as_ref()), + returning: None, + rls_filters: Vec::new(), + })], + lsn: 8, + }, + ); + assert_eq!( + insert.status, + Status::Ok, + "phantom insert must commit: {insert:?}" + ); + + // Validating the (now stale) absent-read against the current write index + // must abort: the insert at LSN 8 is AFTER the read at LSN 5. + let staged = send( + &mut core, + &mut tx, + &mut rx, + stage_static( + 10, + 0, + vec![kv_put("phantomtarget", b"tk", b"tv")], + vec![absent_read], + ), + phantom_vshard, + None, + ); + assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); + assert_eq!( + staged.stage_vote, + Some(StageVote::SerializationConflict), + "a phantom insert into a key observed absent must abort the stage vote" + ); +} + +/// Absent-DOCUMENT phantom safety: a document `PointGet` on an absent document +/// cannot record `Point(KeyRepr::Surrogate(s))`, because the placeholder +/// surrogate `s` a miss carries is allocated from a monotonic counter unrelated +/// to `document_id` — it never coincides with the freshly-minted surrogate a +/// subsequent INSERT of that `document_id` receives, so a per-key OCC check +/// would never catch the phantom. The capture layer therefore degrades an +/// absent document read to `Predicate` (collection floor). A concurrent INSERT +/// into that collection advances `coll_write_lsn` past the read_lsn, so +/// `WriteVersionIndex::read_is_valid` (predicate branch) judges the stale read +/// invalid and the stage VOTES ABORT — collection-granular phantom safety. +#[test] +fn absent_document_phantom_insert_is_caught() { + let (mut core, mut tx, mut rx, _dir) = make_core(); + + let doc_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "phantomdocs") + .vshard() + .as_u32(); + + // The document was absent when read: capture degraded the miss to a + // collection-scoped predicate on "phantomdocs" at read_lsn 5. + let absent_doc_read = VersionedReadEntry { + engine: EngineTag::Document, + collection: "phantomdocs".to_string(), + key: ReadKeyIdent::Predicate, + read_lsn: Lsn::new(5), + home_vshard: None, + served_by: 0, + }; + + // Concurrently, a document with the same document_id the read targeted is + // actually inserted -- allocated a freshly-minted surrogate (42), committing + // at LSN 8. Its collection floor advance (phantomdocs -> 8) is what the + // predicate read validates against. + const NEWLY_ALLOCATED_SURROGATE: u32 = 42; + let insert = commit_calvin( + &mut core, + &mut tx, + &mut rx, + CalvinSeed { + epoch: 1, + vshard: doc_vshard, + collection: "phantomdocs", + plans: vec![doc_insert( + "phantomdocs", + "the-doc-id", + NEWLY_ALLOCATED_SURROGATE, + )], + lsn: 8, + }, + ); + assert_eq!( + insert.status, + Status::Ok, + "phantom insert must commit: {insert:?}" + ); + + // Validate the stale absent-read: the predicate check sees the phantomdocs + // collection floor (LSN 8) is AFTER the read (LSN 5), so the read is no + // longer current and the stage must abort. + let staged = send( + &mut core, + &mut tx, + &mut rx, + stage_static( + 11, + 0, + vec![kv_put("docphantomtarget", b"tk", b"tv")], + vec![absent_doc_read], + ), + doc_vshard, + None, + ); + assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); + assert_eq!( + staged.stage_vote, + Some(StageVote::SerializationConflict), + "a phantom insert into a collection observed absent must abort the stage \ + vote via the collection floor" + ); +} + +/// No-over-abort companion to `absent_document_phantom_insert_is_caught`: the +/// collection-floor degrade must NOT abort every absent-read transaction. An +/// absent document read (predicate on "phantomdocs" at LSN 5) stays valid when +/// no insert lands in THAT collection — a concurrent insert into an UNRELATED +/// collection advances only its own floor, leaving phantomdocs at zero, so the +/// stale read is still current and the stage VOTES COMMIT. +#[test] +fn absent_document_read_without_matching_insert_still_commits() { + let (mut core, mut tx, mut rx, _dir) = make_core(); + + let doc_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "phantomdocs") + .vshard() + .as_u32(); + + let absent_doc_read = VersionedReadEntry { + engine: EngineTag::Document, + collection: "phantomdocs".to_string(), + key: ReadKeyIdent::Predicate, + read_lsn: Lsn::new(5), + home_vshard: None, + served_by: 0, + }; + + // A concurrent insert into a DIFFERENT collection commits at LSN 8. It + // advances only "othercoll"'s floor; phantomdocs is untouched. + const UNRELATED_SURROGATE: u32 = 42; + let insert = commit_calvin( + &mut core, + &mut tx, + &mut rx, + CalvinSeed { + epoch: 1, + vshard: doc_vshard, + collection: "othercoll", + plans: vec![doc_insert("othercoll", "other-id", UNRELATED_SURROGATE)], + lsn: 8, + }, + ); + assert_eq!( + insert.status, + Status::Ok, + "unrelated insert must commit: {insert:?}" + ); + + // The predicate read on phantomdocs is still current: its collection floor + // never advanced past read_lsn 5, so the stage must commit. + let staged = send( + &mut core, + &mut tx, + &mut rx, + stage_static( + 11, + 0, + vec![kv_put("docphantomtarget", b"tk", b"tv")], + vec![absent_doc_read], + ), + doc_vshard, + None, + ); + assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); + assert_eq!( + staged.stage_vote, + Some(StageVote::Commit), + "an absent-document read must NOT over-abort when no insert lands in its \ + collection" + ); +} diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply/read_set.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply/read_set.rs new file mode 100644 index 000000000..eef0184a6 --- /dev/null +++ b/nodedb/tests/inproc/cases/calvin_two_phase_apply/read_set.rs @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The stage vote judges the slice's read-set against the local write +//! versions: a current read commits, a stale one aborts and drops. + +use nodedb::bridge::envelope::{StageVote, Status}; +use nodedb::types::*; +use nodedb_physical::physical_plan::{MetaOp, PhysicalPlan}; +use nodedb_types::calvin::{EngineTag, ReadKeyIdent, VersionedReadEntry}; + +use super::support::*; + +/// An invalid vote (a stale versioned read against a newer local write) STAGES +/// but never applies; the drop discards it and base stays unchanged. +#[test] +fn drop_discards_invalid_staged_calvin_write() { + let (mut core, mut tx, mut rx, _dir) = make_core(); + + // Seed a committed write to `dropcoll` at LSN 100 so its collection write + // version floor is 100. The seed carries a WAL LSN so the version records. + let seed = commit_calvin( + &mut core, + &mut tx, + &mut rx, + CalvinSeed { + epoch: 1, + vshard: 0, + collection: "dropcoll", + plans: vec![kv_put("dropcoll", b"seed", b"v")], + lsn: 100, + }, + ); + assert_eq!(seed.status, Status::Ok, "seed write must commit: {seed:?}"); + + // The read entry's collection must home to the staged request's vShard for + // the read-set check to consider it. + let read_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "dropcoll") + .vshard() + .as_u32(); + + // A read of `dropcoll` observed at LSN 50 — stale against the seed's write + // at LSN 100 → the read-set is no longer current → abort vote. + let stale_read = VersionedReadEntry { + engine: EngineTag::Kv, + collection: "dropcoll".to_string(), + key: ReadKeyIdent::Predicate, + read_lsn: Lsn::new(50), + home_vshard: None, + served_by: 0, + }; + + let staged = send( + &mut core, + &mut tx, + &mut rx, + stage_static( + 7, + 0, + vec![kv_put("targetcoll", b"tk", b"tv")], + vec![stale_read], + ), + read_vshard, + None, + ); + assert_eq!( + staged.status, + Status::Ok, + "stage must succeed even on abort vote" + ); + assert_eq!( + staged.stage_vote, + Some(StageVote::SerializationConflict), + "stale read-set must produce an abort vote" + ); + + // Staged, not applied: the target write is invisible. + let before = send( + &mut core, + &mut tx, + &mut rx, + kv_get("targetcoll", b"tk"), + 0, + None, + ); + assert!( + before.payload.is_empty() || before.status == Status::Error, + "aborted staged write must NOT be visible; got {before:?}" + ); + + // Drop discards the staged plans and fires nothing. The drop must target + // the SAME vShard the stage keyed under (as production dispatches it), so it + // actually pops this participant's staged slice. + let dropped = send( + &mut core, + &mut tx, + &mut rx, + PhysicalPlan::Meta(MetaOp::CalvinDrop { + epoch: 7, + position: 0, + }), + read_vshard, + None, + ); + assert_eq!(dropped.status, Status::Ok, "drop must succeed: {dropped:?}"); + + // Still invisible after the drop — base was never mutated. + let after = send( + &mut core, + &mut tx, + &mut rx, + kv_get("targetcoll", b"tk"), + 0, + None, + ); + assert!( + after.payload.is_empty() || after.status == Status::Error, + "dropped write must never be visible; got {after:?}" + ); +} + +/// A `Point` read (not just a `Predicate` read) at or after the recorded write +/// LSN is current → commit vote, and the stamped install applies the staged +/// write. +#[test] +fn point_read_at_write_lsn_commits_and_the_install_applies() { + let (mut core, mut tx, mut rx, _dir) = make_core(); + + // Seed a committed write to key `pk` in `pointcoll` at LSN 10. + let seed = commit_calvin( + &mut core, + &mut tx, + &mut rx, + CalvinSeed { + epoch: 1, + vshard: 0, + collection: "pointcoll", + plans: vec![kv_put("pointcoll", b"pk", b"v1")], + lsn: 10, + }, + ); + assert_eq!(seed.status, Status::Ok, "seed write must commit: {seed:?}"); + + let point_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "pointcoll") + .vshard() + .as_u32(); + + // A Point read of the exact same key observed at LSN 10 (== the write) is + // still current: no write happened AFTER the read. + let current_read = VersionedReadEntry { + engine: EngineTag::Kv, + collection: "pointcoll".to_string(), + key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"pk".as_slice()))), + read_lsn: Lsn::new(10), + home_vshard: None, + served_by: 0, + }; + + let staged = send( + &mut core, + &mut tx, + &mut rx, + stage_static( + 8, + 0, + vec![kv_put("committarget", b"ck", b"cv")], + vec![current_read], + ), + point_vshard, + None, + ); + assert_eq!(staged.status, Status::Ok, "stage must succeed: {staged:?}"); + assert_eq!( + staged.stage_vote, + Some(StageVote::Commit), + "a read at or after the last write LSN must be current -> commit vote" + ); + + let install_plan = resolved_install(&mut core, &mut tx, &mut rx, 8, point_vshard); + let installed = send( + &mut core, + &mut tx, + &mut rx, + install_plan, + point_vshard, + Some(Lsn::new(80)), + ); + assert_eq!( + installed.status, + Status::Ok, + "install must succeed: {installed:?}" + ); + + let after = send( + &mut core, + &mut tx, + &mut rx, + kv_get("committarget", b"ck"), + 0, + None, + ); + assert_eq!( + after.status, + Status::Ok, + "read after the install must succeed" + ); + assert!( + !after.payload.is_empty(), + "the installed write must be visible after the install" + ); +} + +/// A `Point` read of a key STALE against a later write to that same key +/// (read_lsn=5 vs. a committed write at LSN=10) aborts the stage vote, and the +/// drop discards the staged write with no base mutation — mirrors +/// `drop_discards_invalid_staged_calvin_write` but for a Point key (not a +/// collection-scoped Predicate). +#[test] +fn stale_point_read_of_kv_key_aborts_stage_and_drop_discards() { + let (mut core, mut tx, mut rx, _dir) = make_core(); + + // Seed a committed write to key `pk` in `stalecoll` at LSN 10. + let seed = commit_calvin( + &mut core, + &mut tx, + &mut rx, + CalvinSeed { + epoch: 1, + vshard: 0, + collection: "stalecoll", + plans: vec![kv_put("stalecoll", b"pk", b"v1")], + lsn: 10, + }, + ); + assert_eq!(seed.status, Status::Ok, "seed write must commit: {seed:?}"); + + let stale_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "stalecoll") + .vshard() + .as_u32(); + + // A Point read of the exact same key observed at LSN 5 — stale against the + // write at LSN 10 → the read-set is no longer current → abort vote. + let stale_read = VersionedReadEntry { + engine: EngineTag::Kv, + collection: "stalecoll".to_string(), + key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"pk".as_slice()))), + read_lsn: Lsn::new(5), + home_vshard: None, + served_by: 0, + }; + + let staged = send( + &mut core, + &mut tx, + &mut rx, + stage_static( + 9, + 0, + vec![kv_put("aborttarget", b"ak", b"av")], + vec![stale_read], + ), + stale_vshard, + None, + ); + assert_eq!( + staged.status, + Status::Ok, + "stage must succeed even on abort vote" + ); + assert_eq!( + staged.stage_vote, + Some(StageVote::SerializationConflict), + "a stale Point read (write after read_lsn) must produce an abort vote" + ); + + let before = send( + &mut core, + &mut tx, + &mut rx, + kv_get("aborttarget", b"ak"), + 0, + None, + ); + assert!( + before.payload.is_empty() || before.status == Status::Error, + "aborted staged write must NOT be visible; got {before:?}" + ); + + let dropped = send( + &mut core, + &mut tx, + &mut rx, + PhysicalPlan::Meta(MetaOp::CalvinDrop { + epoch: 9, + position: 0, + }), + stale_vshard, + None, + ); + assert_eq!(dropped.status, Status::Ok, "drop must succeed: {dropped:?}"); + + let after = send( + &mut core, + &mut tx, + &mut rx, + kv_get("aborttarget", b"ak"), + 0, + None, + ); + assert!( + after.payload.is_empty() || after.status == Status::Error, + "dropped write must never be visible; got {after:?}" + ); +} diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply/support.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply/support.rs new file mode 100644 index 000000000..846b6fd45 --- /dev/null +++ b/nodedb/tests/inproc/cases/calvin_two_phase_apply/support.rs @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Helpers every staged Calvin apply case drives a `CoreLoop` with. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use nodedb::bridge::dispatch::{BridgeRequest, BridgeResponse}; +use nodedb::bridge::envelope::{Priority, Request, Response, Status}; +use nodedb::data::executor::core_loop::CoreLoop; +use nodedb::types::*; +use nodedb_bridge::buffer::{Consumer, Producer, RingBuffer}; +use nodedb_physical::physical_plan::{ + CalvinInstall, CalvinResolved, DocumentOp, KvOp, MetaOp, PhysicalPlan, RedoOrigin, +}; +use nodedb_types::QualifiedCollection; +use nodedb_types::Surrogate; +use nodedb_types::calvin::VersionedReadEntry; + +pub(super) fn make_core() -> ( + CoreLoop, + Producer, + Consumer, + tempfile::TempDir, +) { + let dir = tempfile::tempdir().unwrap(); + let (req_tx, req_rx) = RingBuffer::channel::(64); + let (resp_tx, resp_rx) = RingBuffer::channel::(64); + let core = CoreLoop::open( + 0, + req_rx, + resp_tx, + dir.path(), + Arc::new(nodedb_types::OrdinalClock::new()), + nodedb::data::executor::core_loop::test_governor(), + ) + .unwrap(); + (core, req_tx, resp_rx, dir) +} + +/// Build a request for `plan` on `vshard`, carrying an optional committed WAL +/// LSN (present on the seed write so its version is recorded). +pub(super) fn make_request(plan: PhysicalPlan, vshard: u32, wal_lsn: Option) -> Request { + Request { + request_id: RequestId::new(1), + tenant_id: TenantId::new(1), + vshard_id: VShardId::new(vshard), + database_id: DatabaseId::DEFAULT, + plan, + deadline: Instant::now() + Duration::from_secs(5), + priority: Priority::Normal, + trace_id: nodedb_types::TraceId::ZERO, + consistency: ReadConsistency::Strong, + idempotency_key: None, + event_source: nodedb::event::EventSource::RaftFollower, + user_roles: Vec::new(), + user_id: None, + statement_digest: None, + txn_id: None, + wal_lsn, + resolved_now_ms: None, + commit_hlc: None, + admission: nodedb::bridge::envelope::Admission::Admitted, + } +} + +pub(super) fn send( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, + plan: PhysicalPlan, + vshard: u32, + wal_lsn: Option, +) -> Response { + tx.try_push(BridgeRequest::unfloored(make_request( + plan, vshard, wal_lsn, + ))) + .unwrap(); + core.tick(); + rx.try_pop().unwrap().inner +} + +/// Resolve the transaction staged at `(epoch, 0)` on `vshard` and build the +/// stamped install of its redo record. +pub(super) fn resolved_install( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, + epoch: u64, + vshard: u32, +) -> PhysicalPlan { + let resolved = send( + core, + tx, + rx, + PhysicalPlan::Meta(MetaOp::CalvinResolve { epoch, position: 0 }), + vshard, + None, + ); + assert_eq!( + resolved.status, + Status::Ok, + "resolve must succeed: {resolved:?}" + ); + stamped_install(epoch, &resolved, Vec::new()) +} + +/// The stamped install of the slice at `(epoch, 0)` whose `CalvinResolve` +/// answered `resolved`. +pub(super) fn stamped_install( + epoch: u64, + resolved: &Response, + collections: Vec, +) -> PhysicalPlan { + let answer: CalvinResolved = + zerompk::from_msgpack(resolved.payload.as_bytes()).expect("decode resolved answer"); + PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { + redo: answer.redo, + collections, + sum_targets: Vec::new(), + origin: RedoOrigin::Commit, + calvin: Some(CalvinInstall { + epoch, + position: 0, + epoch_system_ms: 0, + reply: answer.reply, + user_write: true, + }), + }) +} + +/// A write committed through the Calvin path to seed a write version. +pub(super) struct CalvinSeed<'a> { + pub(super) epoch: u64, + pub(super) vshard: u32, + /// The collection `plans` write; the install records its floor at `lsn`. + pub(super) collection: &'a str, + pub(super) plans: Vec, + pub(super) lsn: u64, +} + +/// Commit `seed.plans` as the Calvin transaction at `(seed.epoch, 0)` on +/// `seed.vshard` and install its redo record at `seed.lsn`: the path a +/// committed multi-shard write takes. The install records each written key +/// and the collection floor at that LSN. +pub(super) fn commit_calvin( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, + seed: CalvinSeed<'_>, +) -> Response { + let staged = send( + core, + tx, + rx, + stage_static(seed.epoch, 0, seed.plans, Vec::new()), + seed.vshard, + None, + ); + assert_eq!( + staged.status, + Status::Ok, + "seed stage must succeed: {staged:?}" + ); + let resolved = send( + core, + tx, + rx, + PhysicalPlan::Meta(MetaOp::CalvinResolve { + epoch: seed.epoch, + position: 0, + }), + seed.vshard, + None, + ); + assert_eq!( + resolved.status, + Status::Ok, + "seed resolve must succeed: {resolved:?}" + ); + send( + core, + tx, + rx, + stamped_install(seed.epoch, &resolved, vec![seed.collection.to_string()]), + seed.vshard, + Some(Lsn::new(seed.lsn)), + ) +} + +pub(super) fn kv_put(coll: &str, key: &[u8], value: &[u8]) -> PhysicalPlan { + PhysicalPlan::Kv(KvOp::Put { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), + key: key.to_vec(), + value: value.to_vec(), + ttl_ms: 0, + surrogate: nodedb_test_support::kv_rows::kv_row_surrogate(key), + returning: None, + rls_filters: Vec::new(), + provenance: None, + }) +} + +pub(super) fn kv_get(coll: &str, key: &[u8]) -> PhysicalPlan { + PhysicalPlan::Kv(KvOp::Get { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), + key: key.to_vec(), + rls_filters: Vec::new(), + surrogate_ceiling: None, + }) +} + +/// A minimal msgpack-encoded document body, `{"a": "1"}`. +pub(super) fn doc_value() -> Vec { + let mut obj = std::collections::HashMap::new(); + obj.insert("a".to_string(), nodedb_types::Value::String("1".into())); + zerompk::to_msgpack_vec(&nodedb_types::Value::Object(obj)).unwrap() +} + +/// A minimal document `PointInsert` plan, used to record a write-version at a +/// specific surrogate (the document engine's per-key identity). +pub(super) fn doc_insert(coll: &str, document_id: &str, surrogate: u32) -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::PointInsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, coll), + document_id: document_id.to_string(), + value: doc_value(), + if_absent: false, + surrogate: Surrogate::new(surrogate), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: Vec::new(), + }) +} + +pub(super) fn stage_static( + epoch: u64, + position: u32, + plans: Vec, + versioned_reads: Vec, +) -> PhysicalPlan { + PhysicalPlan::Meta(MetaOp::CalvinExecuteStatic { + epoch, + position, + tenant_id: TenantId::new(1), + plans, + epoch_system_ms: 0, + versioned_reads, + body_plans: Vec::new(), + }) +} + +/// Push one prebuilt request through the ring and return its response. +pub(super) fn send_request( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, + request: Request, +) -> Response { + tx.try_push(BridgeRequest::unfloored(request)).unwrap(); + core.tick(); + rx.try_pop().unwrap().inner +} diff --git a/nodedb/tests/wire/cases/sql_backup_calvin_cut.rs b/nodedb/tests/wire/cases/sql_backup_calvin_cut.rs index 2eac7356b..2d64c00de 100644 --- a/nodedb/tests/wire/cases/sql_backup_calvin_cut.rs +++ b/nodedb/tests/wire/cases/sql_backup_calvin_cut.rs @@ -3,11 +3,12 @@ //! A backup's consistent cut covers Calvin transactions. //! //! A transaction that writes two collections on two vShards commits through -//! the Calvin scheduler. Its install records the transaction's commit HLC on -//! the tenant's write mark before the COMMIT is acknowledged, so a Calvin -//! commit after a backup refuses a restore of it. A Calvin transaction held -//! at its flush while a backup starts was sequenced before the backup's cut, -//! so the backup waits for its install and holds its rows. +//! the Calvin scheduler. The data-group apply of its stamped redo records the +//! transaction's commit HLC on the tenant's write mark before the COMMIT is +//! acknowledged, so a Calvin commit after a backup refuses a restore of it. A +//! Calvin transaction whose redo proposal is held while a backup starts was +//! sequenced before the backup's cut, so the backup waits for its install and +//! holds its rows. use super::backup_support::{drain_backup, names_on_two_vshards, push_restore}; use crate::harness::TestServer; @@ -65,30 +66,32 @@ async fn a_calvin_commit_after_the_backup_refuses_the_restore() { "expected the staleness refusal, got: {error}" ); assert!( - error.contains("calvin flush"), - "the refusal must name the Calvin commit as the newer write, got: {error}" + error.contains("replicated apply"), + "the refusal must name the Calvin commit's data-group apply as the newer write, \ + got: {error}" ); } #[cfg(feature = "failpoints")] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn a_calvin_commit_held_at_its_flush_is_in_the_backup_or_refuses_its_restore() { +async fn a_calvin_commit_held_at_its_redo_is_in_the_backup_or_refuses_its_restore() { use std::time::Duration; const PARKED_FOR: Duration = Duration::from_millis(1500); let (first, second) = names_on_two_vshards("calvin_cut"); let gate_dir = tempfile::tempdir().expect("gate tempdir"); - let release = gate_dir.path().join("release-flush"); + let release = gate_dir.path().join("release-redo"); let server = TestServer::start_with_failpoints(&format!( - "calvin::before_flush::{first}=wait_file({})", + "calvin::before_redo_propose::{first}=wait_file({})", release.display() )) .await; create_kv(&server, &first).await; create_kv(&server, &second).await; - // Hold the Calvin transaction at its flush on `first`'s vShard. + // Hold the redo proposal of the Calvin transaction's slice on `first`'s + // vShard. let (committer, committer_handle) = server .connect_as("nodedb", "nodedb") .await @@ -106,7 +109,7 @@ async fn a_calvin_commit_held_at_its_flush_is_in_the_backup_or_refuses_its_resto tokio::time::sleep(PARKED_FOR).await; assert!( !commit.is_finished(), - "the COMMIT was not held at its flush" + "the COMMIT was not held at its redo proposal" ); // Back up while the transaction is sequenced but not installed. @@ -121,7 +124,7 @@ async fn a_calvin_commit_held_at_its_flush_is_in_the_backup_or_refuses_its_resto "the backup snapshotted before a Calvin transaction sequenced ahead of its cut installed" ); - std::fs::write(&release, b"release").expect("release the flush"); + std::fs::write(&release, b"release").expect("release the redo proposal"); commit .await .expect("commit task") diff --git a/nodedb/tests/wire/cases/sql_backup_restore_durable_marks.rs b/nodedb/tests/wire/cases/sql_backup_restore_durable_marks.rs index 1f5f8847d..27171307a 100644 --- a/nodedb/tests/wire/cases/sql_backup_restore_durable_marks.rs +++ b/nodedb/tests/wire/cases/sql_backup_restore_durable_marks.rs @@ -103,7 +103,7 @@ async fn a_calvin_commit_after_the_backup_refuses_the_restore_after_a_restart() "expected the staleness refusal, got: {error}" ); assert!( - error.contains("calvin flush"), + error.contains("replicated apply"), "the refusal must name the Calvin commit as the newer write, got: {error}" ); } From 4e13c654e02510ec3b5634d6e15f2c936583d581 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sat, 10 Oct 2026 05:47:29 +0800 Subject: [PATCH 06/19] refactor(calvin): mark slices applied through stamped redo records A Calvin slice is now recorded as applied by the stamped redo record that installs it, so the separate CalvinApplied WAL record and the flush path that wrote it are removed. The scheduler restart path, the reply handlers, and the planner and admission code follow the single install path. CDC positions drop the per-transaction base component: a replicated event is positioned by its (epoch, index) alone, and the position partition type is removed. --- .../cases/calvin_cluster_pgwire_e2e.rs | 2 +- ...ti_shard_bitemporal_best_effort_restart.rs | 4 +- .../calvin_multi_shard_bitemporal_restart.rs | 6 +- .../cases/calvin_multi_shard_redo_restart.rs | 2 +- .../cases/calvin_multishard_pk_read.rs | 2 +- .../cases/cross_node_read_occ_abort.rs | 2 +- .../cases/gather_join_in_txn_occ.rs | 2 +- .../cases/native_gather_join_in_txn_occ.rs | 2 +- .../cases/shuffle_aggregate_in_txn_occ.rs | 2 +- .../cases/shuffle_join_in_txn_occ.rs | 2 +- .../single_node_calvin_write_versions.rs | 11 +- .../src/physical_plan/collection.rs | 28 +- nodedb-physical/src/physical_plan/meta.rs | 29 +- nodedb-wal/src/lib.rs | 4 +- nodedb-wal/src/record/calvin.rs | 107 ----- nodedb-wal/src/record/mod.rs | 4 - nodedb-wal/src/record/types.rs | 22 +- .../bootstrap/write_group_settle/settle.rs | 25 -- .../control/change_stream/journal/codec.rs | 12 +- .../control/change_stream/journal/store.rs | 7 +- .../src/control/change_stream/stream/bus.rs | 8 +- .../control/change_stream/stream/cursor.rs | 47 +- .../control/change_stream/stream/fanout.rs | 23 +- .../change_stream/stream/journaling.rs | 29 +- .../src/control/change_stream/stream/ring.rs | 2 +- nodedb/src/control/checkpoint_manager.rs | 5 +- nodedb/src/control/checkpoint_task.rs | 4 +- .../calvin/scheduler/driver/core/scheduler.rs | 2 +- .../scheduler/driver/core/test_support.rs | 2 +- .../control/cluster/calvin/scheduler/mod.rs | 4 +- .../cluster/calvin/scheduler/recovery.rs | 295 +++++-------- nodedb/src/control/planner/calvin/dispatch.rs | 2 +- .../control/planner/calvin/edge_sequencing.rs | 7 +- .../calvin/tx_class/write_keys/graph.rs | 4 +- .../calvin/tx_class/write_keys/plan.rs | 2 +- .../src/control/planner/rls_injection/meta.rs | 1 - .../rls_injection/permission_tree/meta.rs | 1 - .../security/identity/plan_permission.rs | 1 - .../dispatch_utils/change_events/publish.rs | 2 +- .../submit_write/funnel/driver.rs | 7 +- .../server/exchange/all_cores/dispatch.rs | 1 - .../src/control/server/http/routes/health.rs | 10 +- .../control/server/shared/returning/inject.rs | 1 - .../server/shared/session/commit_calvin.rs | 7 +- .../src/control/server/shared/session/live.rs | 2 +- .../shared/write_admission/admission_keys.rs | 1 - .../predicate/txn_buffering/classify.rs | 8 - .../server/shared/write_admission/route.rs | 125 +----- nodedb/src/control/state/calvin_ack_result.rs | 6 +- nodedb/src/control/state/calvin_local.rs | 2 +- nodedb/src/control/state/tenant_marks.rs | 14 +- .../wal_replication/transaction_redo/apply.rs | 1 - nodedb/src/control/write_resolve/mod.rs | 4 +- .../src/control/write_resolve/timeseries.rs | 39 -- nodedb/src/data/executor/core_loop/state.rs | 2 +- nodedb/src/data/executor/dispatch/meta.rs | 19 +- .../handlers/control/calvin/active_passive.rs | 12 +- .../executor/handlers/control/calvin/flush.rs | 400 ------------------ .../executor/handlers/control/calvin/mod.rs | 4 +- .../handlers/control/calvin/shared.rs | 8 +- .../handlers/control/calvin/static_stage.rs | 2 +- .../handlers/control/calvin/test_commit.rs | 48 ++- .../handlers/control/calvin_active_verify.rs | 2 +- .../handlers/control/calvin_overlay_stage.rs | 10 +- .../control/calvin_overlay_stage_bulk.rs | 10 +- .../handlers/control/calvin_reply/mod.rs | 2 +- .../{flush_read.rs => payload.rs} | 8 +- .../handlers/control/calvin_reply/reply.rs | 12 +- .../handlers/control/calvin_reply/stage.rs | 13 +- .../handlers/control/calvin_resolve.rs | 4 +- .../timeseries/ingest_resolved_outcome.rs | 2 +- .../redo_apply/calvin_install_tests.rs | 175 +++++++- nodedb/src/event/cdc/consume/remote.rs | 3 +- nodedb/src/event/cdc/consume/sink.rs | 11 +- nodedb/src/event/cdc/offset.rs | 43 +- nodedb/src/event/cdc/position/ledger.rs | 56 +-- nodedb/src/event/cdc/position/marker.rs | 16 - nodedb/src/event/cdc/position/mod.rs | 7 +- nodedb/src/event/cdc/position/partition.rs | 35 -- nodedb/src/event/cdc/position/sequencer.rs | 168 +------- nodedb/src/event/cdc/router.rs | 55 +-- nodedb/src/event/cross_shard/types.rs | 6 +- nodedb/src/event/topic/committed/deliver.rs | 1 - nodedb/src/event/topic/committed/key.rs | 16 +- nodedb/src/event/trigger/lane/fire.rs | 2 +- nodedb/src/event/trigger/lane/held.rs | 8 +- nodedb/src/event/trigger/lane/position.rs | 31 +- nodedb/src/event/trigger/lane/snapshot.rs | 34 +- nodedb/src/event/wal_replay.rs | 10 - nodedb/src/event/wal_replay_kind.rs | 6 - nodedb/src/main_boot/shutdown_wiring.rs | 2 +- nodedb/src/wal/manager/append_metadata.rs | 28 +- nodedb/src/wal/redo/record.rs | 11 +- nodedb/src/wal/replay/surrogate/dispatch.rs | 1 - nodedb/src/wal/replay/sync_hwm/advance.rs | 1 - .../inproc/cases/calvin_scheduler_restart.rs | 122 +++--- 96 files changed, 644 insertions(+), 1706 deletions(-) delete mode 100644 nodedb-wal/src/record/calvin.rs delete mode 100644 nodedb/src/data/executor/handlers/control/calvin/flush.rs rename nodedb/src/data/executor/handlers/control/calvin_reply/{flush_read.rs => payload.rs} (94%) delete mode 100644 nodedb/src/event/cdc/position/partition.rs diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_cluster_pgwire_e2e.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_cluster_pgwire_e2e.rs index cb4625379..35f5385da 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_cluster_pgwire_e2e.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_cluster_pgwire_e2e.rs @@ -165,7 +165,7 @@ async fn calvin_multishard_write_in_explicit_block_commits() { before={admitted_before} after={admitted_after}" ); - // Both rows are readable after the commit applied. The Calvin flush lands + // Both rows are readable after the commit applied. The redo install lands // asynchronously after the completion ack, so poll for visibility. let deadline = std::time::Instant::now() + Duration::from_secs(10); loop { diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_bitemporal_best_effort_restart.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_bitemporal_best_effort_restart.rs index 650e59af1..a3a051e47 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_bitemporal_best_effort_restart.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_bitemporal_best_effort_restart.rs @@ -175,7 +175,7 @@ async fn calvin_multi_shard_bitemporal_best_effort_commit_survives_wal_only_rest // ONE `simple_query` call carrying the whole transaction: the two INSERTs // are buffered during the block, and on COMMIT `classify_dispatch` sees // the full task set spanning two vShards → MultiShard → best-effort fans - // out into two independent single-vShard Calvin flushes. + // out into two independent single-vShard Calvin transactions. let txn_sql = format!( "BEGIN; \ INSERT INTO {coll_a} (id, value) VALUES ('a1', 'va'); \ @@ -199,7 +199,7 @@ async fn calvin_multi_shard_bitemporal_best_effort_commit_survives_wal_only_rest .await; // Pre-restart: both rows are visible and each carries exactly one - // system-time version. The Calvin flush lands asynchronously after the + // system-time version. The redo install lands asynchronously after the // completion ack, so poll. let (stamp_a, stamp_b); let deadline = std::time::Instant::now() + Duration::from_secs(10); diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_bitemporal_restart.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_bitemporal_restart.rs index 67d690f1c..3214c4a7a 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_bitemporal_restart.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_bitemporal_restart.rs @@ -171,8 +171,8 @@ async fn calvin_multi_shard_bitemporal_commit_survives_wal_only_restart() { // ONE `simple_query` call carrying the whole transaction: the two INSERTs // are buffered during the block, and on COMMIT `classify_dispatch` sees - // the full task set spanning two vShards → MultiShard → leader-routed - // Calvin flush. + // the full task set spanning two vShards → MultiShard → one Calvin + // transaction. let txn_sql = format!( "BEGIN; \ INSERT INTO {coll_a} (id, value) VALUES ('a1', 'va'); \ @@ -193,7 +193,7 @@ async fn calvin_multi_shard_bitemporal_commit_survives_wal_only_restart() { .await; // Pre-restart: both rows are visible and each carries exactly one - // system-time version. The Calvin flush lands asynchronously after the + // system-time version. The redo install lands asynchronously after the // completion ack, so poll. let (stamp_a, stamp_b); let deadline = std::time::Instant::now() + Duration::from_secs(10); diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_redo_restart.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_redo_restart.rs index 921409a28..c0a212ac2 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_redo_restart.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multi_shard_redo_restart.rs @@ -180,7 +180,7 @@ async fn calvin_multi_shard_write_in_explicit_block_commits_and_survives_restart .await; // Pre-restart: both rows are visible and the vector is in the live HNSW. - // The Calvin flush lands asynchronously after the completion ack, so poll. + // The redo install lands asynchronously after the completion ack, so poll. let deadline = std::time::Instant::now() + Duration::from_secs(10); loop { let kv_v = read_once_a_leader_exists( diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multishard_pk_read.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multishard_pk_read.rs index cfbe7fd9b..6790a08e1 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_multishard_pk_read.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_multishard_pk_read.rs @@ -43,7 +43,7 @@ async fn cross_shard_transaction_commit_reports_commit_and_rows_land() { assert_eq!( tags(&mut conn, "COMMIT").await, vec!["COMMIT"], - "the Calvin flush answers exactly one COMMIT tag" + "the Calvin commit answers exactly one COMMIT tag" ); fx.converge().await; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/cross_node_read_occ_abort.rs b/nodedb-cluster-tests/tests/common_suite/cases/cross_node_read_occ_abort.rs index 3c3818403..109d52bb7 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/cross_node_read_occ_abort.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/cross_node_read_occ_abort.rs @@ -355,7 +355,7 @@ async fn cross_shard_read_occ_commits_when_read_still_current() { ) .await; - // Both committed writes become visible (the Calvin flush lands asynchronously + // Both committed writes become visible (the redo install lands asynchronously // after the completion ack, so poll). wait_for_async( "both committed writes visible after COMMIT", diff --git a/nodedb-cluster-tests/tests/common_suite/cases/gather_join_in_txn_occ.rs b/nodedb-cluster-tests/tests/common_suite/cases/gather_join_in_txn_occ.rs index 593526524..560499088 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/gather_join_in_txn_occ.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/gather_join_in_txn_occ.rs @@ -301,7 +301,7 @@ async fn gather_join_commits_when_neither_side_concurrently_written() { ) .await; - // Both committed writes become visible (Calvin flush lands asynchronously). + // Both committed writes become visible (the redo install lands asynchronously). wait_for_async( "both committed writes visible after COMMIT", Duration::from_secs(10), diff --git a/nodedb-cluster-tests/tests/common_suite/cases/native_gather_join_in_txn_occ.rs b/nodedb-cluster-tests/tests/common_suite/cases/native_gather_join_in_txn_occ.rs index a77327353..047bd61d7 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/native_gather_join_in_txn_occ.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/native_gather_join_in_txn_occ.rs @@ -344,7 +344,7 @@ async fn native_gather_join_commits_when_neither_side_concurrently_written() { ) .await; - // Both committed writes become visible (Calvin flush lands asynchronously). + // Both committed writes become visible (the redo install lands asynchronously). wait_for_async( "both committed writes visible after COMMIT", Duration::from_secs(10), diff --git a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_aggregate_in_txn_occ.rs b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_aggregate_in_txn_occ.rs index 3ee2bdb35..5d5926494 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_aggregate_in_txn_occ.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_aggregate_in_txn_occ.rs @@ -246,7 +246,7 @@ async fn shuffle_aggregate_read_commits_when_not_concurrently_written() { ) .await; - // Both committed writes become visible (Calvin flush lands asynchronously). + // Both committed writes become visible (the redo install lands asynchronously). wait_for_async( "both committed writes visible after COMMIT", Duration::from_secs(10), diff --git a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_in_txn_occ.rs b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_in_txn_occ.rs index 4a96ac84c..3c5c75d43 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_in_txn_occ.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_in_txn_occ.rs @@ -273,7 +273,7 @@ async fn shuffle_join_commits_when_neither_side_concurrently_written() { ) .await; - // Both committed writes become visible (Calvin flush lands asynchronously). + // Both committed writes become visible (the redo install lands asynchronously). wait_for_async( "both committed writes visible after COMMIT", Duration::from_secs(10), diff --git a/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_write_versions.rs b/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_write_versions.rs index 0bbdf751d..bf72e193c 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_write_versions.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/single_node_calvin_write_versions.rs @@ -59,8 +59,8 @@ fn distinct_vshard_node_keys() -> (String, String) { } /// A cross-shard (dual-home edge) Calvin-committed write increments the -/// node-global write-version-recorded counter — i.e. its keys' versions are -/// recorded at the CalvinApplied WAL LSN instead of being silently dropped. +/// node-global write-version-recorded counter: the stamped redo install of +/// each slice records its keys' versions at the install's WAL LSN. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn cross_shard_calvin_write_records_write_version() { // 4 Data-Plane cores so distinct vShards land on distinct cores — a genuine @@ -96,7 +96,8 @@ async fn cross_shard_calvin_write_records_write_version() { // AUTOCOMMIT cross-shard edge insert: because the endpoints home to distinct // vShards, the edge dual-homes atomically through the Calvin scheduler // (`submit_calvin_routed`), not the single-home fast path. On commit each - // participating vShard's scheduler records its slice's write versions. + // participating vShard's data group installs its slice's stamped redo, + // and the install records the slice's write versions. node.client .simple_query(&format!( "GRAPH INSERT EDGE IN 'sncalvin_wv_graph' FROM '{src}' TO '{dst}' TYPE 'l'" @@ -104,8 +105,8 @@ async fn cross_shard_calvin_write_records_write_version() { .await .expect("cross-shard edge insert must commit via the single-node Calvin path"); - // Recording is dispatched fire-and-forget after the CalvinApplied WAL LSN is - // obtained, so wait for the counter to advance. + // Each participant's data-group apply loop raises the counter on its own + // schedule, so wait for it to advance. wait_for( "calvin write-version recording fired for the cross-shard commit", Duration::from_secs(10), diff --git a/nodedb-physical/src/physical_plan/collection.rs b/nodedb-physical/src/physical_plan/collection.rs index 403ad813c..02a6c8598 100644 --- a/nodedb-physical/src/physical_plan/collection.rs +++ b/nodedb-physical/src/physical_plan/collection.rs @@ -158,15 +158,14 @@ impl PhysicalPlan { /// Every user collection this plan names: each collection a committed /// redo install writes, or else the one [`Self::collection`] reports. /// - /// A committed-redo apply and a Calvin flush install one record that can - /// write several collections, so [`Self::collection`] reports none for - /// them. A caller that keys on a collection name uses this instead. + /// A committed-redo apply installs one record that can write several + /// collections, so [`Self::collection`] reports none for it. A caller that + /// keys on a collection name uses this instead. pub fn named_collections(&self) -> Vec<&str> { match self { - PhysicalPlan::Meta( - MetaOp::ApplyTransactionRedo { collections, .. } - | MetaOp::CalvinFlush { collections, .. }, - ) => collections.iter().map(String::as_str).collect(), + PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { collections, .. }) => { + collections.iter().map(String::as_str).collect() + } PhysicalPlan::Meta(MetaOp::RestoreRedo(batch)) => { batch.collections.iter().map(String::as_str).collect() } @@ -187,22 +186,13 @@ mod tests { let collections = vec!["a".to_string(), "b".to_string()]; let redo = PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { redo: Vec::new(), - collections: collections.clone(), + collections, sum_targets: Vec::new(), origin: crate::physical_plan::RedoOrigin::Commit, calvin: None, }); - let flush = PhysicalPlan::Meta(MetaOp::CalvinFlush { - epoch: 1, - position: 0, - redo: Vec::new(), - collections, - sum_targets: Vec::new(), - }); - for plan in [redo, flush] { - assert_eq!(plan.collection(), None); - assert_eq!(plan.named_collections(), vec!["a", "b"]); - } + assert_eq!(redo.collection(), None); + assert_eq!(redo.named_collections(), vec!["a", "b"]); } #[test] diff --git a/nodedb-physical/src/physical_plan/meta.rs b/nodedb-physical/src/physical_plan/meta.rs index 0190fab5f..8f29bffbc 100644 --- a/nodedb-physical/src/physical_plan/meta.rs +++ b/nodedb-physical/src/physical_plan/meta.rs @@ -32,7 +32,7 @@ pub enum MetaOp { /// A transaction's plans as one batch. An embedded (Lite) engine executes /// the sub-plans atomically. An Origin core refuses it: a committed /// transaction installs there only through its redo record - /// (`ApplyTransactionRedo`, `CalvinFlush`). + /// (`ApplyTransactionRedo`). /// /// `txn_id` identifies the committing session transaction whose staging /// overlay holds the resolve-time bitemporal stamps an install must reuse. @@ -493,27 +493,6 @@ pub enum MetaOp { savepoint: u64, }, - /// Install a committed Calvin transaction's redo record on base storage. - /// - /// `CalvinExecuteStatic` STAGES the transaction's plans without mutating - /// base, and `CalvinResolve` resolves them into one redo record, which the - /// scheduler appends to the WAL as a `TransactionRedo` record. This op - /// carries that record's bytes, and the request carries its LSN. The core - /// installs it through the same passes restart replay drives: validate, - /// install with undo, then settle and cover. It then drops the staged - /// state keyed by `(epoch, position)`. - /// - /// `redo` is empty when the transaction wrote nothing. `collections` names - /// every collection the transaction wrote. `sum_targets` is the - /// materialized-sum resolution its document writes fold into. - CalvinFlush { - epoch: u64, - position: u32, - redo: Vec, - collections: Vec, - sum_targets: Vec, - }, - /// Discard the staged writes of a Calvin transaction. /// /// Dispatched by the leader's scheduler when the slice ends with no log @@ -554,8 +533,10 @@ pub enum MetaOp { /// `commit_pending` under `(epoch, position, vshard)` and the per-core /// staging overlay written under the corresponding synthetic `TxnId` /// (see `calvin_synthetic_txn_id`). Dispatched by the scheduler once the - /// global verdict is commit, ahead of `CalvinFlush`, which installs the - /// record this op returns. No base engine is touched during resolve. + /// global verdict is commit. The scheduler proposes the record this op + /// returns as a stamped `TransactionRedo` entry, and + /// `ApplyTransactionRedo` installs it. No base engine is touched during + /// resolve. /// /// The response payload is a zerompk-encoded [`super::CalvinResolved`]: /// the redo record and the reply the staged plans decided. diff --git a/nodedb-wal/src/lib.rs b/nodedb-wal/src/lib.rs index 8fb9fc1d9..40c321eba 100644 --- a/nodedb-wal/src/lib.rs +++ b/nodedb-wal/src/lib.rs @@ -87,8 +87,8 @@ pub use preamble::{ pub use reader::{StopReason, WalReader}; #[cfg(not(target_arch = "wasm32"))] pub use record::{ - CalvinAppliedPayload, FtsDeletePayload, FtsIndexPayload, RecordStamp, RecordTarget, RecordType, - SpatialDeletePayload, SpatialPutPayload, WalRecord, WalRecordArgs, WriteAbortedPayload, + FtsDeletePayload, FtsIndexPayload, RecordStamp, RecordTarget, RecordType, SpatialDeletePayload, + SpatialPutPayload, WalRecord, WalRecordArgs, WriteAbortedPayload, }; pub use record::{NO_EVENT_SOURCE, RecordHeader}; #[cfg(not(target_arch = "wasm32"))] diff --git a/nodedb-wal/src/record/calvin.rs b/nodedb-wal/src/record/calvin.rs deleted file mode 100644 index 542126688..000000000 --- a/nodedb-wal/src/record/calvin.rs +++ /dev/null @@ -1,107 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Calvin scheduler WAL payload types. -//! -//! Uses fixed little-endian encoding (no msgpack framing) for zero-allocation -//! replay, consistent with other WAL payload types in this crate. - -use crate::error::{Result, WalError}; - -/// Fixed size of a `CalvinApplied` payload on disk. -/// -/// Layout: `epoch (u64 LE) | position (u32 LE) | vshard_id (u32 LE)` = 16 bytes. -pub const CALVIN_APPLIED_PAYLOAD_SIZE: usize = 16; - -/// Payload for [`super::types::RecordType::CalvinApplied`]. -/// -/// Written by the Calvin executor after a `MetaOp::CalvinExecute` batch -/// commits successfully. Encodes `{ epoch, position, vshard_id }` so the -/// scheduler's restart path can scan the WAL and find `last_applied_epoch` -/// for a given vshard without reading the full Raft log. -/// -/// Fixed little-endian encoding — no msgpack framing. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct CalvinAppliedPayload { - /// Sequencer epoch of the applied transaction. - pub epoch: u64, - /// Zero-based position within the epoch batch. - pub position: u32, - /// The vshard that applied this transaction. - pub vshard_id: u32, -} - -impl CalvinAppliedPayload { - /// Construct a new payload. - pub fn new(epoch: u64, position: u32, vshard_id: u32) -> Self { - Self { - epoch, - position, - vshard_id, - } - } - - /// Serialize to fixed-size bytes for WAL append. - /// - /// Layout: `epoch (8 LE) | position (4 LE) | vshard_id (4 LE)`. - pub fn to_bytes(&self) -> [u8; CALVIN_APPLIED_PAYLOAD_SIZE] { - let mut buf = [0u8; CALVIN_APPLIED_PAYLOAD_SIZE]; - buf[..8].copy_from_slice(&self.epoch.to_le_bytes()); - buf[8..12].copy_from_slice(&self.position.to_le_bytes()); - buf[12..16].copy_from_slice(&self.vshard_id.to_le_bytes()); - buf - } - - /// Deserialize from bytes read from the WAL. - pub fn from_bytes(bytes: &[u8]) -> Result { - if bytes.len() < CALVIN_APPLIED_PAYLOAD_SIZE { - return Err(WalError::InvalidPayload { - detail: format!( - "CalvinApplied payload must be {CALVIN_APPLIED_PAYLOAD_SIZE} bytes, got {}", - bytes.len() - ), - }); - } - let epoch = u64::from_le_bytes(bytes[..8].try_into().expect( - "invariant: bounds-checked above (bytes.len() >= CALVIN_APPLIED_PAYLOAD_SIZE >= 8)", - )); - let position = u32::from_le_bytes(bytes[8..12].try_into().expect( - "invariant: bounds-checked above (bytes.len() >= CALVIN_APPLIED_PAYLOAD_SIZE >= 12)", - )); - let vshard_id = u32::from_le_bytes(bytes[12..16].try_into().expect( - "invariant: bounds-checked above (bytes.len() >= CALVIN_APPLIED_PAYLOAD_SIZE >= 16)", - )); - Ok(Self { - epoch, - position, - vshard_id, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn roundtrip() { - let p = CalvinAppliedPayload::new(42, 7, 3); - let bytes = p.to_bytes(); - let decoded = CalvinAppliedPayload::from_bytes(&bytes).unwrap(); - assert_eq!(decoded.epoch, 42); - assert_eq!(decoded.position, 7); - assert_eq!(decoded.vshard_id, 3); - } - - #[test] - fn too_short_payload_returns_error() { - let result = CalvinAppliedPayload::from_bytes(&[0u8; 5]); - assert!(result.is_err()); - } - - #[test] - fn payload_size_is_correct() { - let p = CalvinAppliedPayload::new(u64::MAX, u32::MAX, u32::MAX); - let bytes = p.to_bytes(); - assert_eq!(bytes.len(), CALVIN_APPLIED_PAYLOAD_SIZE); - } -} diff --git a/nodedb-wal/src/record/mod.rs b/nodedb-wal/src/record/mod.rs index 280e177ea..ce0bebbfc 100644 --- a/nodedb-wal/src/record/mod.rs +++ b/nodedb-wal/src/record/mod.rs @@ -5,8 +5,6 @@ pub mod aborted; #[cfg(not(target_arch = "wasm32"))] pub mod anchor; #[cfg(not(target_arch = "wasm32"))] -pub mod calvin; -#[cfg(not(target_arch = "wasm32"))] pub mod fts_spatial; pub mod header; #[cfg(not(target_arch = "wasm32"))] @@ -27,8 +25,6 @@ pub use aborted::{WRITE_ABORTED_PAYLOAD_SIZE, WriteAbortedPayload}; #[cfg(not(target_arch = "wasm32"))] pub use anchor::{TIME_ANCHOR_PAYLOAD_SIZE, TimeAnchorPayload}; #[cfg(not(target_arch = "wasm32"))] -pub use calvin::CalvinAppliedPayload; -#[cfg(not(target_arch = "wasm32"))] pub use fts_spatial::{FtsDeletePayload, FtsIndexPayload, SpatialDeletePayload, SpatialPutPayload}; pub use header::{ ENCRYPTED_FLAG, HEADER_SIZE, MAX_WAL_PAYLOAD_SIZE, NO_EVENT_SOURCE, RecordHeader, diff --git a/nodedb-wal/src/record/types.rs b/nodedb-wal/src/record/types.rs index 9fb4907e3..48334772f 100644 --- a/nodedb-wal/src/record/types.rs +++ b/nodedb-wal/src/record/types.rs @@ -195,8 +195,9 @@ pub enum RecordType { /// uses, so replay reconstitutes a `WalRecord` per sub-op and feeds it to /// that engine's existing replay path — no tag loss, no re-encoding. /// - /// May also carry a Calvin stamp so a cross-shard transaction's durable - /// record doubles as its sequencer applied-marker. + /// May also carry a Calvin stamp. A stamped record marks its Calvin slice + /// applied on its vShard. + /// /// Payload: zerompk-encoded `RedoRecord` (see the `wal::redo` module). /// /// Required: skipping this record on replay would drop a committed @@ -258,21 +259,6 @@ pub enum RecordType { /// versions resurrected and diverge from the leader's state. TemporalPurge = 103 | 0x8000, - /// Calvin scheduler: marks a sequenced transaction as applied on this - /// vshard. - /// - /// Written by the Calvin executor after a `MetaOp::CalvinExecute` batch - /// commits successfully. The scheduler's rebuild path scans the WAL for - /// these records to determine `last_applied_epoch` on restart. - /// - /// Payload: zerompk-encoded `CalvinAppliedPayload { epoch: u64, - /// position: u32, vshard_id: u32 }`. - /// - /// Required: a replay that skipped this record would leave the scheduler - /// believing the transaction was not applied and re-dispatch it after - /// restart, causing double-application. - CalvinApplied = 110 | 0x8000, - /// Sync idempotency watermark — advances the durable per-stream /// high-watermark for a given producer so the receiver can safely /// deduplicate re-delivered ingest frames on reconnect. @@ -478,7 +464,6 @@ impl RecordType { x if x == 101 | 0x8000 => Some(Self::CollectionTombstoned), 102 => Some(Self::TimeAnchor), x if x == 103 | 0x8000 => Some(Self::TemporalPurge), - x if x == 110 | 0x8000 => Some(Self::CalvinApplied), x if x == 53 | 0x8000 => Some(Self::SyncSeqAdvance), x if x == 54 | 0x8000 => Some(Self::FtsIndex), x if x == 55 | 0x8000 => Some(Self::FtsDelete), @@ -572,7 +557,6 @@ mod tests { RecordType::CollectionTombstoned, RecordType::TimeAnchor, RecordType::TemporalPurge, - RecordType::CalvinApplied, RecordType::SyncSeqAdvance, RecordType::FtsIndex, RecordType::FtsDelete, diff --git a/nodedb/src/bootstrap/write_group_settle/settle.rs b/nodedb/src/bootstrap/write_group_settle/settle.rs index 87420f5da..bb06bed83 100644 --- a/nodedb/src/bootstrap/write_group_settle/settle.rs +++ b/nodedb/src/bootstrap/write_group_settle/settle.rs @@ -18,8 +18,6 @@ //! transaction record is the exception: it always applies, so it stays, //! its group closes with one part with no rows, and replay runs its //! install. -//! - A committed Calvin record is whole at append and takes no part. Its -//! stored write set holds no row, so boot journals nothing for it. //! //! Restart replay, a point-in-time restore of the archived WAL, and the event //! stream rebuilt from it then agree with the stores. Boot makes the appended @@ -157,11 +155,6 @@ fn complete_group( scanned: &WalGroups, capture: &WriteSetCapture, ) -> crate::Result { - // A Calvin flush's origin is a committed record whole at append: no part - // follows it. - if is_calvin_flush(&stored_plan(&capture.origin_append)?) { - return Ok(false); - } let appender = part_appender(wal, &capture.origin_append)?; let target = target(capture, Lsn::new(capture.origin)); let write_set = capture.write_set(); @@ -188,16 +181,6 @@ fn journal_cut_write(wal: &WalManager, capture: &WriteSetCapture) -> crate::Resu let inputs = &capture.origin_append; let appender = part_appender(wal, inputs)?; let plan = stored_plan(inputs)?; - // A Calvin flush's origin is the transaction's committed record. The - // sequencer holds the transaction, and its recovery runs it again. - if is_calvin_flush(&plan) { - tracing::warn!( - origin = capture.origin, - "boot found a Calvin flush whose committed record the crash cut; the \ - sequencer runs the transaction again" - ); - return Ok(false); - } let home = target(capture, Lsn::ZERO); if let Some((epoch, group_id, log_index)) = inputs.change_position { let marker = ChangePositionMarker { @@ -244,14 +227,6 @@ fn stored_plan(inputs: &OriginAppend) -> crate::Result { }) } -/// Whether `plan` is a Calvin flush, whose origin is whole at append. -fn is_calvin_flush(plan: &PhysicalPlan) -> bool { - matches!( - plan, - PhysicalPlan::Meta(nodedb_physical::physical_plan::MetaOp::CalvinFlush { .. }) - ) -} - /// The appender every record of the write carried. fn part_appender<'a>(wal: &'a WalManager, inputs: &OriginAppend) -> crate::Result> { let appender = wal diff --git a/nodedb/src/control/change_stream/journal/codec.rs b/nodedb/src/control/change_stream/journal/codec.rs index b66e97ca6..1266ef63f 100644 --- a/nodedb/src/control/change_stream/journal/codec.rs +++ b/nodedb/src/control/change_stream/journal/codec.rs @@ -18,17 +18,14 @@ pub(super) const ROW_KEY_LEN: usize = PARTITION_KEY_LEN + POSITION_LEN; const FEED_LEN: usize = 2 * POSITION_LEN + 8; const TAG_GROUP: u8 = 0; -const TAG_CALVIN: u8 = 1; fn word(bytes: &[u8], at: usize) -> Option { Some(u64::from_be_bytes(bytes.get(at..at + 8)?.try_into().ok()?)) } pub(super) fn partition_key(partition: ChangePartition) -> [u8; PARTITION_KEY_LEN] { - let (tag, id) = match partition { - ChangePartition::Group(group) => (TAG_GROUP, group), - ChangePartition::Calvin(vshard) => (TAG_CALVIN, u64::from(vshard)), - }; + let ChangePartition(group) = partition; + let (tag, id) = (TAG_GROUP, group); let mut out = [0u8; PARTITION_KEY_LEN]; out[0] = tag; out[1..].copy_from_slice(&id.to_be_bytes()); @@ -38,8 +35,7 @@ pub(super) fn partition_key(partition: ChangePartition) -> [u8; PARTITION_KEY_LE pub(super) fn decode_partition(bytes: &[u8]) -> Option { let id = word(bytes, 1)?; match *bytes.first()? { - TAG_GROUP => Some(ChangePartition::Group(id)), - TAG_CALVIN => u32::try_from(id).ok().map(ChangePartition::Calvin), + TAG_GROUP => Some(ChangePartition(id)), _ => None, } } @@ -159,7 +155,7 @@ mod tests { #[test] fn keys_and_feeds_round_trip_and_sort_by_position() { - for partition in [ChangePartition::Group(7), ChangePartition::Calvin(3)] { + for partition in [ChangePartition(7), ChangePartition(3)] { assert_eq!(decode_partition(&partition_key(partition)), Some(partition)); let low = row_key(partition, CdcOffset::data_event(0, 5, 1)); let high = row_key(partition, CdcOffset::data_event(0, 5, 2)); diff --git a/nodedb/src/control/change_stream/journal/store.rs b/nodedb/src/control/change_stream/journal/store.rs index 255befe65..4ce448642 100644 --- a/nodedb/src/control/change_stream/journal/store.rs +++ b/nodedb/src/control/change_stream/journal/store.rs @@ -9,9 +9,8 @@ //! - A data group's changes are journaled when the apply loop saves the //! group's applied floor, before the floor: every entry at or below a //! saved floor has its changes on disk. Entries above it are delivered -//! again after a restart and publish again. -//! - A Calvin transaction's changes are journaled before its applied marker -//! is durable. +//! again after a restart and publish again. A committed Calvin slice +//! installs from a data-group entry, so its changes follow this rule. //! - A forwarded run is journaled before the receiver acks it. //! //! Each partition keeps `after`, the position above which it holds every @@ -199,7 +198,7 @@ mod tests { use crate::control::change_stream::{ChangeEvent, ChangeOperation}; use crate::types::{DatabaseId, Lsn, TenantId}; - const GROUP: ChangePartition = ChangePartition::Group(2); + const GROUP: ChangePartition = ChangePartition(2); fn event(index: u64) -> SequencedChangeEvent { SequencedChangeEvent::new( diff --git a/nodedb/src/control/change_stream/stream/bus.rs b/nodedb/src/control/change_stream/stream/bus.rs index 5cc496569..6b489ddda 100644 --- a/nodedb/src/control/change_stream/stream/bus.rs +++ b/nodedb/src/control/change_stream/stream/bus.rs @@ -193,7 +193,7 @@ impl ChangeStream { } } let run = ChangeRun { - partition: ChangePartition::Group(group_id), + partition: ChangePartition(group_id), after: Some(CdcOffset::whole_index(first.saturating_sub(1))), through: CdcOffset::whole_index(last), changes, @@ -213,7 +213,7 @@ impl ChangeStream { .staged .retain(|&(group, index), _| group != group_id || index > last_included_index); state.ring.raise_floor( - ChangePartition::Group(group_id), + ChangePartition(group_id), CdcOffset::whole_index(last_included_index), ); } @@ -597,7 +597,7 @@ mod tests { let outsider = ChangeStream::new(16); let mut live = outsider.subscribe(None, Some(TenantId::new(1))); let mut cursor = live.start_cursor().clone(); - let group = ChangePartition::Group(3); + let group = ChangePartition(3); let deliver = |run: &AppendedRun| { outsider.deliver_remote_notify(&NotifyBroadcastMsg::from_run(1, run)) }; @@ -642,7 +642,7 @@ mod tests { let outsider = ChangeStream::new(16); let mut live = outsider.subscribe(None, Some(TenantId::new(1))); let mut cursor = live.start_cursor().clone(); - let group = ChangePartition::Group(5); + let group = ChangePartition(5); let deliver = |run: &AppendedRun| { outsider.deliver_remote_notify(&NotifyBroadcastMsg::from_run(1, run)) }; diff --git a/nodedb/src/control/change_stream/stream/cursor.rs b/nodedb/src/control/change_stream/stream/cursor.rs index db4236687..40870f97c 100644 --- a/nodedb/src/control/change_stream/stream/cursor.rs +++ b/nodedb/src/control/change_stream/stream/cursor.rs @@ -3,11 +3,9 @@ //! Resumable positions in the Control-Plane change stream. //! //! The stream is a set of partitions. Each partition is one totally ordered -//! feed that every node numbers alike: -//! -//! - `Group(g)`: the writes of data group `g`, at their Raft log index. -//! - `Calvin(v)`: the Calvin transactions vShard `v` applied, at their -//! sequencer position. +//! feed that every node numbers alike. `ChangePartition(g)` holds the writes +//! of data group `g`, at their Raft log index. A committed Calvin slice is one +//! of them: it installs from a data-group entry. //! //! A cursor holds, per partition, the position of the last event it //! consumed. A cursor taken on one node therefore resumes on any node that @@ -22,8 +20,7 @@ use crate::event::cdc::CdcOffset; use super::SequencedChangeEvent; const TOKEN_PREFIX: &str = "v2:"; -/// Partitions one cursor holds at most: every data group and every vShard's -/// Calvin feed of a large cluster. +/// Partitions one cursor holds at most: every data group of a large cluster. const MAX_ENTRIES: usize = 8192; /// Upper bound on one entry's text: a tag, a 20-digit id, and three /// 20-digit numbers with their separators. @@ -32,12 +29,10 @@ const MAX_TOKEN_LEN: usize = TOKEN_PREFIX.len() + MAX_ENTRIES * (MAX_ENTRY_LEN + /// One totally ordered feed of the change stream. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum ChangePartition { - /// A data group's writes, at their Raft log index. - Group(u64), - /// A vShard's Calvin transactions, at their sequencer position. - Calvin(u32), -} +pub struct ChangePartition( + /// The data group whose writes the partition holds, at their Raft log index. + pub u64, +); /// What a consumer does with the next live event. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -112,10 +107,8 @@ impl fmt::Display for ChangeCursor { if index > 0 { formatter.write_str(",")?; } - match partition { - ChangePartition::Group(group) => write!(formatter, "g{group}")?, - ChangePartition::Calvin(vshard) => write!(formatter, "c{vshard}")?, - } + let ChangePartition(group) = partition; + write!(formatter, "g{group}")?; write!( formatter, "@{}.{}.{}", @@ -167,10 +160,7 @@ impl FromStr for ChangeCursor { fn parse_entry(entry: &str) -> Result<(ChangePartition, CdcOffset), CursorParseError> { let (partition, position) = entry.split_once('@').ok_or(CursorParseError)?; let partition = match partition.as_bytes().first() { - Some(b'g') => ChangePartition::Group(number(&partition[1..])?), - Some(b'c') => ChangePartition::Calvin( - u32::try_from(number(&partition[1..])?).map_err(|_| CursorParseError)?, - ), + Some(b'g') => ChangePartition(number(&partition[1..])?), _ => return Err(CursorParseError), }; let mut parts = position.split('.'); @@ -228,11 +218,8 @@ mod tests { #[test] fn token_round_trips_and_is_strict() { let mut cursor = ChangeCursor::default(); - cursor.raise(ChangePartition::Group(3), CdcOffset::data_event(0, 42, 1)); - cursor.raise( - ChangePartition::Calvin(7), - CdcOffset::data_event_in(0, 9, 2, 1), - ); + cursor.raise(ChangePartition(3), CdcOffset::data_event(0, 42, 1)); + cursor.raise(ChangePartition(7), CdcOffset::data_event(0, 9, 1)); let token = cursor.to_string(); assert!(token.starts_with("v2:")); assert_eq!(token.parse::(), Ok(cursor)); @@ -245,7 +232,7 @@ mod tests { "v2:x1@0.1.2", "v2:l@0.1.2", "v2:l7@0.1.2", - "v2:c99999999999@0.1.2", + "v2:c1@0.1.2", "v2:g1@0.-1.2", ] { assert!(invalid.parse::().is_err(), "{invalid}"); @@ -254,7 +241,7 @@ mod tests { #[test] fn accept_delivers_new_events_and_skips_covered_ones() { - let group = ChangePartition::Group(1); + let group = ChangePartition(1); let mut cursor = ChangeCursor::default(); let first = event(group, CdcOffset::data_event(0, 5, 1), CdcOffset::ZERO); let second = event(group, CdcOffset::data_event(0, 6, 1), CdcOffset::ZERO); @@ -264,7 +251,7 @@ mod tests { assert!(cursor.covers(&second)); // Another partition's positions never compare with this one. let other = event( - ChangePartition::Group(2), + ChangePartition(2), CdcOffset::data_event(0, 1, 1), CdcOffset::ZERO, ); @@ -273,7 +260,7 @@ mod tests { #[test] fn a_hole_above_the_cursor_resets() { - let group = ChangePartition::Group(1); + let group = ChangePartition(1); let mut cursor = ChangeCursor::default(); cursor.raise(group, CdcOffset::whole_index(10)); let past_hole = event( diff --git a/nodedb/src/control/change_stream/stream/fanout.rs b/nodedb/src/control/change_stream/stream/fanout.rs index cef387379..79d854014 100644 --- a/nodedb/src/control/change_stream/stream/fanout.rs +++ b/nodedb/src/control/change_stream/stream/fanout.rs @@ -25,9 +25,7 @@ use tokio::sync::mpsc::error::TrySendError; use tracing::{trace, warn}; use crate::control::state::SharedState; -use crate::event::cross_shard::types::{ - NOTIFY_PARTITION_CALVIN, NOTIFY_PARTITION_GROUP, NotifyBroadcastMsg, NotifyChange, -}; +use crate::event::cross_shard::types::{NOTIFY_PARTITION_GROUP, NotifyBroadcastMsg, NotifyChange}; use super::ChangePartition; use super::ring::AppendedRun; @@ -51,13 +49,7 @@ impl ChangeFanout { return Vec::new(); }; let routing = routing.read().unwrap_or_else(|p| p.into_inner()); - let group = match partition { - ChangePartition::Group(group) => group, - ChangePartition::Calvin(vshard) => match routing.group_for_vshard(vshard) { - Ok(group) => group, - Err(_) => return Vec::new(), - }, - }; + let ChangePartition(group) = partition; let Some(info) = routing.group_info(group) else { return Vec::new(); }; @@ -176,10 +168,8 @@ fn spawn_peer( impl NotifyBroadcastMsg { /// The wire form of `run`. pub(crate) fn from_run(source_node: u64, run: &AppendedRun) -> Self { - let (partition_kind, partition_id) = match run.partition { - ChangePartition::Group(group) => (NOTIFY_PARTITION_GROUP, group), - ChangePartition::Calvin(vshard) => (NOTIFY_PARTITION_CALVIN, u64::from(vshard)), - }; + let ChangePartition(group) = run.partition; + let (partition_kind, partition_id) = (NOTIFY_PARTITION_GROUP, group); Self { source_node, partition_kind, @@ -206,10 +196,7 @@ impl NotifyBroadcastMsg { /// The partition the run belongs to, `None` for an unknown kind. pub fn partition(&self) -> Option { match self.partition_kind { - NOTIFY_PARTITION_GROUP => Some(ChangePartition::Group(self.partition_id)), - NOTIFY_PARTITION_CALVIN => u32::try_from(self.partition_id) - .ok() - .map(ChangePartition::Calvin), + NOTIFY_PARTITION_GROUP => Some(ChangePartition(self.partition_id)), _ => None, } } diff --git a/nodedb/src/control/change_stream/stream/journaling.rs b/nodedb/src/control/change_stream/stream/journaling.rs index 687042771..15f470bde 100644 --- a/nodedb/src/control/change_stream/stream/journaling.rs +++ b/nodedb/src/control/change_stream/stream/journaling.rs @@ -30,7 +30,7 @@ impl ChangeStream { /// feeds and are journaled with the next batch of their feed. pub fn attach_journal(&self, journal: Arc) -> crate::Result<()> { let feeds = journal.load()?; - let early_runs = { + { let mut state = self.lock(); let (early_events, early_feeds) = state.ring.drain(); for feed in feeds { @@ -41,7 +41,6 @@ impl ChangeStream { changes: feed.changes, }); } - let mut early_runs = Vec::new(); for (partition, floor, through) in early_feeds { let changes = early_events .iter() @@ -58,25 +57,19 @@ impl ChangeStream { through, changes, }); - match partition { - ChangePartition::Group(group_id) => state - .group_pending - .entry(group_id) - .or_default() - .extend(appended.events), - ChangePartition::Calvin(_) => early_runs.push(appended), - } + let ChangePartition(group_id) = partition; + state + .group_pending + .entry(group_id) + .or_default() + .extend(appended.events); } - early_runs - }; + } self.journal .set(journal) .map_err(|_| crate::Error::Internal { detail: "the change-feed journal was already attached".into(), })?; - for run in &early_runs { - self.journal_run(run); - } Ok(()) } @@ -98,7 +91,7 @@ impl ChangeStream { /// covers then has its changes on disk. pub(crate) fn journal_group(&self, group_id: u64, floor: u64) { let through = CdcOffset::whole_index(floor); - let partition = ChangePartition::Group(group_id); + let partition = ChangePartition(group_id); let batch = { let mut state = self.lock(); if self.journal.get().is_none() { @@ -120,8 +113,8 @@ impl ChangeStream { self.persist(batch); } - /// Journal a run a Calvin transaction or a forwarding leader produced, - /// before its publisher acknowledges it. + /// Journal a run a forwarding leader produced, before its publisher + /// acknowledges it. pub(super) fn journal_run(&self, run: &AppendedRun) { if self.journal.get().is_none() { return; diff --git a/nodedb/src/control/change_stream/stream/ring.rs b/nodedb/src/control/change_stream/stream/ring.rs index fe5b5084d..b33cc570a 100644 --- a/nodedb/src/control/change_stream/stream/ring.rs +++ b/nodedb/src/control/change_stream/stream/ring.rs @@ -352,7 +352,7 @@ mod tests { use crate::control::change_stream::ChangeOperation; use crate::types::Lsn; - const GROUP: ChangePartition = ChangePartition::Group(4); + const GROUP: ChangePartition = ChangePartition(4); fn change(index: u64, ordinal: u64) -> PositionedChange { PositionedChange { diff --git a/nodedb/src/control/checkpoint_manager.rs b/nodedb/src/control/checkpoint_manager.rs index 8a71cc7f7..86e27c30d 100644 --- a/nodedb/src/control/checkpoint_manager.rs +++ b/nodedb/src/control/checkpoint_manager.rs @@ -174,7 +174,8 @@ pub struct CheckpointCycleInputs<'a> { /// When present, the tombstone set is GC'd to the new truncation point. pub catalog: Option<&'a crate::control::security::catalog::SystemCatalog>, /// The Calvin applied ledger of every vShard on this node. Saved in - /// `catalog` before truncation deletes the applied markers it came from. + /// `catalog` before truncation deletes the stamped redo records it came + /// from. pub calvin_ledgers: Option<&'a crate::control::cluster::calvin::scheduler::CalvinAppliedLedgers>, } @@ -403,7 +404,7 @@ pub async fn run_checkpoint_cycle(inputs: CheckpointCycleInputs<'_>) -> Option (Scheduler, tempfile::Temp sequencer_proposer: CapturingProposer::accepting(), sequencer_state_machine, // A freshly-built scheduler has applied nothing, so its ledger holds the - // not-yet-applied sentinel (matching `read_applied_recovery` for a clean + // not-yet-applied sentinel (matching `recover_all_applied` for a clean // node). A watermark of `0` will instead claim epoch 0 is fully applied, // making the exactly-once gate (`AppliedGate::is_applied`) short-circuit // every epoch-0 replay before it reaches the lock table — silently diff --git a/nodedb/src/control/cluster/calvin/scheduler/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/mod.rs index 4e4fd83d6..a36daf75d 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/mod.rs @@ -24,6 +24,4 @@ pub use lock::{AcquireOutcome, HotKeyTable, LockKey, LockManager, LockMode, TxnI // keep that path stable via an alias while the module lives under `lock/`. pub use lock as lock_manager; pub use metrics::SchedulerMetrics; -pub use recovery::{ - AppliedRecovery, NOT_YET_APPLIED_EPOCH, read_applied_recovery, recover_all_applied, -}; +pub use recovery::{AppliedRecovery, NOT_YET_APPLIED_EPOCH, recover_all_applied}; diff --git a/nodedb/src/control/cluster/calvin/scheduler/recovery.rs b/nodedb/src/control/cluster/calvin/scheduler/recovery.rs index 46e13c864..8b1779c27 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/recovery.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/recovery.rs @@ -2,12 +2,12 @@ //! Calvin scheduler WAL recovery. //! -//! Provides [`read_applied_recovery`] which scans the WAL for -//! `RecordType::CalvinApplied` records AND `RecordType::TransactionRedo` -//! records carrying a `calvin_stamp` (write-bearing Calvin transactions -//! journal the redo record as their applied-marker instead of a standalone -//! `CalvinApplied` marker), returning for a given vShard the union of applied -//! `(epoch, position)` pairs together with a fully-applied watermark. +//! Provides [`recover_all_applied`], which scans the WAL for +//! `RecordType::TransactionRedo` records carrying a `calvin_stamp`. A +//! committed Calvin slice installs from a stamped redo entry, and its WAL +//! record is the slice's applied marker. The scan returns, per vShard, the +//! applied `(epoch, position)` pairs together with a fully-applied +//! watermark. //! //! Boot runs [`recover_all_applied`] once, before the data-group apply loop //! starts: one WAL pass yields every vShard's applied state, merged with the @@ -16,7 +16,7 @@ //! its exactly-once applied gate (see [`super::applied_gate::AppliedGate`]) //! from its vShard's ledger. //! -//! Each `CalvinApplied` marker is per `(epoch, position, vShard)` — one per +//! Each stamp is per `(epoch, position, vShard)` — one per //! independent transaction position — so the scan preserves `position` rather //! than collapsing an epoch to a single "applied" bit. Collapsing to the max //! epoch will mark a whole epoch applied on the strength of its first committed @@ -33,8 +33,8 @@ use std::collections::{BTreeMap, BTreeSet}; +use nodedb_wal::WalRecord; use nodedb_wal::record::RecordType; -use nodedb_wal::{CalvinAppliedPayload, WalRecord}; use tracing::warn; use crate::wal::RedoRecord; @@ -44,7 +44,7 @@ use crate::wal::manager::WalManager; /// exists for this vShard. pub const NOT_YET_APPLIED_EPOCH: u64 = u64::MAX; -/// Result of scanning the WAL for a vShard's `CalvinApplied` markers. +/// Result of scanning the WAL for a vShard's stamped redo records. #[derive(Debug, Clone, PartialEq, Eq)] pub struct AppliedRecovery { /// Fully-applied watermark `W` to seed the applied gate. Conservative at @@ -79,51 +79,28 @@ impl AppliedRecovery { } } -/// Scan the WAL and collect this vShard's applied `(epoch, position)` markers. +/// Every vShard's applied `(epoch, position)` markers in one pass over the +/// WAL. `group_of` names a vShard's data group: a snapshot install of the +/// group drops the vShard's markers before it. The install replaced the +/// vShard's storage, and the applied state it brought replaced this node's +/// in the catalog. /// -/// Returns an empty tail and the [`NOT_YET_APPLIED_EPOCH`] sentinel for a -/// greenfield node (no `CalvinApplied` records exist). +/// A vShard with no stamped redo record is absent from the result. /// /// Records that fail to decode are logged and skipped — a corrupt record does /// not abort the scan. -pub fn read_applied_recovery(wal: &WalManager, vshard_id: u32) -> crate::Result { - scan_applied(wal, vshard_id, None) -} - -/// [`read_applied_recovery`] for a vShard of data group `group_id`: a -/// snapshot install of the group drops every marker before it. The install -/// replaced the vShard's storage, and the applied state it brought replaced -/// this node's in the catalog. -fn scan_applied( - wal: &WalManager, - vshard_id: u32, - group_id: Option, -) -> crate::Result { - let group_of = |vshard: u32| if vshard == vshard_id { group_id } else { None }; - let mut scans = scan_all(wal, &group_of, Some(vshard_id))?; - Ok(scans - .remove(&vshard_id) - .unwrap_or_else(AppliedRecovery::empty)) -} - -/// Every vShard's applied markers in one pass over the WAL. `only` limits -/// the pass to one vShard. `group_of` names a vShard's data group: a -/// snapshot install of the group drops the vShard's markers before it. fn scan_all( wal: &WalManager, group_of: &dyn Fn(u32) -> Option, - only: Option, ) -> crate::Result> { let records = wal.replay()?; let mut scans: BTreeMap = BTreeMap::new(); let note = |scans: &mut BTreeMap, vshard_id: u32, epoch: u64, position: u32| { - if only.is_none_or(|wanted| wanted == vshard_id) { - scans - .entry(vshard_id) - .or_insert_with(AppliedRecovery::empty) - .note(epoch, position); - } + scans + .entry(vshard_id) + .or_insert_with(AppliedRecovery::empty) + .note(epoch, position); }; for record in &records { @@ -141,18 +118,6 @@ fn scan_all( } } } - Some(RecordType::CalvinApplied) => { - match CalvinAppliedPayload::from_bytes(&record.payload) { - Ok(p) => note(&mut scans, p.vshard_id, p.epoch, p.position), - Err(e) => { - warn!( - lsn = record.header.lsn, - error = %e, - "calvin recovery: failed to decode CalvinApplied payload; skipping" - ); - } - } - } Some(RecordType::TransactionRedo) => match RedoRecord::from_bytes(&record.payload) { Ok(redo) => { if let Some(stamp) = redo.calvin_stamp { @@ -191,7 +156,7 @@ pub fn recover_all_applied( catalog: &crate::control::security::catalog::SystemCatalog, group_of: &dyn Fn(u32) -> Option, ) -> crate::Result> { - let mut recovered = scan_all(wal, group_of, None)?; + let mut recovered = scan_all(wal, group_of)?; for saved in catalog.load_all_calvin_applied()? { let from_wal = recovered .remove(&saved.vshard_id) @@ -247,17 +212,70 @@ mod tests { use super::*; use tempfile::TempDir; + use crate::types::{DatabaseId, TenantId, VShardId}; use crate::wal::manager::WalManager; + use crate::wal::{CalvinStamp, RedoRecord, RedoSubRecord}; fn open_wal(dir: &TempDir) -> WalManager { WalManager::open(dir.path(), false).expect("open wal") } + /// A redo record with one sub-record and the Calvin stamp `stamp`. + fn redo_record(stamp: Option) -> RedoRecord { + RedoRecord { + version: 1, + ops: vec![RedoSubRecord { + record_type: nodedb_wal::record::RecordType::Put as u32, + payload: vec![1, 2, 3], + }], + calvin_stamp: stamp, + cross_shard_applied: None, + row_sources: Vec::new(), + publishes: Vec::new(), + row_changes: Vec::new(), + } + } + + fn append_redo(wal: &WalManager, vshard: u32, record: &RedoRecord) { + wal.appender(crate::wal::manager::NO_APPLY_KEY) + .with_event_source(crate::event::EventSource::User) + .append_transaction_redo( + TenantId::new(0), + VShardId::new(vshard), + DatabaseId::DEFAULT, + record, + ) + .expect("append redo"); + } + + /// Append the stamped redo record of the Calvin slice at + /// `(epoch, position)` on `vshard`. + fn append_stamped(wal: &WalManager, vshard: u32, epoch: u64, position: u32) { + let stamp = CalvinStamp { + epoch, + position, + vshard_id: vshard, + }; + append_redo(wal, vshard, &redo_record(Some(stamp))); + } + + /// The recovered state of `vshard` alone, from a catalog with nothing + /// saved. `group_id` is the data group of `vshard`, when known. A vShard + /// with no marker recovers as `AppliedRecovery::empty()`. + fn recover_vshard(wal: &WalManager, vshard: u32, group_id: Option) -> AppliedRecovery { + let catalog = crate::control::security::catalog::SystemCatalog::open_in_memory().unwrap(); + let group_of = |v: u32| if v == vshard { group_id } else { None }; + recover_all_applied(wal, &catalog, &group_of) + .unwrap() + .remove(&vshard) + .unwrap_or_else(AppliedRecovery::empty) + } + #[test] fn greenfield_returns_sentinel_and_empty_tail() { let dir = TempDir::new().unwrap(); let wal = open_wal(&dir); - let rec = read_applied_recovery(&wal, 1).unwrap(); + let rec = recover_vshard(&wal, 1, None); assert_eq!(rec.fully_applied_epoch, NOT_YET_APPLIED_EPOCH); assert_eq!(rec.max_applied_epoch, NOT_YET_APPLIED_EPOCH); assert!(rec.applied_tail.is_empty()); @@ -268,21 +286,14 @@ mod tests { let dir = TempDir::new().unwrap(); let wal = open_wal(&dir); - use crate::types::VShardId; // Epoch 5: position 0 applied, position 1 NOT applied. Epoch 2: pos 0. - wal.appender(crate::wal::manager::NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(1), 2, 0) - .unwrap(); - wal.appender(crate::wal::manager::NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(1), 5, 0) - .unwrap(); + append_stamped(&wal, 1, 2, 0); + append_stamped(&wal, 1, 5, 0); // A different vshard (must be ignored). - wal.appender(crate::wal::manager::NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(2), 99, 0) - .unwrap(); + append_stamped(&wal, 2, 99, 0); wal.sync().unwrap(); - let rec = read_applied_recovery(&wal, 1).unwrap(); + let rec = recover_vshard(&wal, 1, None); // The recovery API reports (5,0) applied and (5,1) NOT applied — the // exact per-position distinction the old max-epoch collapse destroyed. assert!(rec.applied_tail.contains(&(5, 0)), "(5,0) is applied"); @@ -294,7 +305,7 @@ mod tests { assert_eq!(rec.fully_applied_epoch, NOT_YET_APPLIED_EPOCH); assert!(rec.fully_applied_epoch == NOT_YET_APPLIED_EPOCH || rec.fully_applied_epoch < 5); - let rec2 = read_applied_recovery(&wal, 2).unwrap(); + let rec2 = recover_vshard(&wal, 2, None); assert!(rec2.applied_tail.contains(&(99, 0))); assert_eq!(rec2.max_applied_epoch, 99); } @@ -303,17 +314,14 @@ mod tests { fn multi_position_epoch_is_not_collapsed() { let dir = TempDir::new().unwrap(); let wal = open_wal(&dir); - use crate::types::VShardId; let vshard = 3u32; // Epoch 7 carries two independent positions on this vShard; only // position 0 committed before the crash. - wal.appender(crate::wal::manager::NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard), 7, 0) - .unwrap(); + append_stamped(&wal, vshard, 7, 0); wal.sync().unwrap(); - let rec = read_applied_recovery(&wal, vshard).unwrap(); + let rec = recover_vshard(&wal, vshard, None); assert!(rec.applied_tail.contains(&(7, 0))); assert!( !rec.applied_tail.contains(&(7, 1)), @@ -323,84 +331,20 @@ mod tests { } #[test] - fn transaction_redo_calvin_stamp_unions_with_calvin_applied() { - use crate::types::{DatabaseId, TenantId, VShardId}; - use crate::wal::{CalvinStamp, RedoRecord, RedoSubRecord}; - + fn only_stamped_transaction_redo_records_mark_positions_applied() { let dir = TempDir::new().unwrap(); let wal = open_wal(&dir); let vshard = 4u32; - // A pure-read/empty-ops txn still writes a standalone CalvinApplied - // marker at (epoch 1, position 0). - wal.appender(crate::wal::manager::NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard), 1, 0) - .unwrap(); - - // A write-bearing Calvin txn journals its applied-marker as a - // TransactionRedo record carrying a calvin_stamp at (epoch 1, position 1). - let write_bearing = RedoRecord { - version: 1, - ops: vec![RedoSubRecord { - record_type: nodedb_wal::record::RecordType::Put as u32, - payload: vec![1, 2, 3], - }], - calvin_stamp: Some(CalvinStamp { - epoch: 1, - position: 1, - vshard_id: vshard, - }), - cross_shard_applied: None, - row_sources: Vec::new(), - publishes: Vec::new(), - row_changes: Vec::new(), - }; - wal.appender(crate::wal::manager::NO_APPLY_KEY) - .with_event_source(crate::event::EventSource::User) - .append_transaction_redo( - TenantId::new(0), - VShardId::new(vshard), - DatabaseId::DEFAULT, - &write_bearing, - ) - .unwrap(); - - // A single-shard TransactionRedo (calvin_stamp: None) must be ignored - // by Calvin recovery. - let single_shard = RedoRecord { - version: 1, - ops: vec![RedoSubRecord { - record_type: nodedb_wal::record::RecordType::Put as u32, - payload: vec![9, 9, 9], - }], - calvin_stamp: None, - cross_shard_applied: None, - row_sources: Vec::new(), - publishes: Vec::new(), - row_changes: Vec::new(), - }; - wal.appender(crate::wal::manager::NO_APPLY_KEY) - .with_event_source(crate::event::EventSource::User) - .append_transaction_redo( - TenantId::new(0), - VShardId::new(vshard), - DatabaseId::DEFAULT, - &single_shard, - ) - .unwrap(); - + append_stamped(&wal, vshard, 1, 0); + append_stamped(&wal, vshard, 1, 1); + // A session commit's record carries no stamp, and Calvin recovery + // ignores it. + append_redo(&wal, vshard, &redo_record(None)); wal.sync().unwrap(); - let rec = read_applied_recovery(&wal, vshard).unwrap(); - assert!( - rec.applied_tail.contains(&(1, 0)), - "CalvinApplied marker still contributes" - ); - assert!( - rec.applied_tail.contains(&(1, 1)), - "TransactionRedo calvin_stamp contributes its (epoch, position) too" - ); - assert_eq!(rec.applied_tail.len(), 2); + let rec = recover_vshard(&wal, vshard, None); + assert_eq!(rec.applied_tail, [(1, 0), (1, 1)].into_iter().collect()); assert_eq!(rec.max_applied_epoch, 1); } @@ -408,11 +352,8 @@ mod tests { fn saved_state_restores_what_a_truncated_wal_lost() { let dir = TempDir::new().unwrap(); let wal = open_wal(&dir); - use crate::types::VShardId; - // The WAL still holds only the marker written after the checkpoint. - wal.appender(crate::wal::manager::NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(1), 6, 0) - .unwrap(); + // The WAL still holds only the record written after the checkpoint. + append_stamped(&wal, 1, 6, 0); wal.sync().unwrap(); let catalog_dir = TempDir::new().unwrap(); let catalog = crate::control::security::catalog::SystemCatalog::open( @@ -449,27 +390,16 @@ mod tests { /// alone yields. #[test] fn one_pass_recovery_matches_per_vshard_scans() { - use crate::types::VShardId; let dir = TempDir::new().unwrap(); let wal = open_wal(&dir); let appender = || wal.appender(crate::wal::manager::NO_APPLY_KEY); - appender() - .append_calvin_applied(VShardId::new(1), 1, 0) - .unwrap(); - appender() - .append_calvin_applied(VShardId::new(2), 1, 1) - .unwrap(); + append_stamped(&wal, 1, 1, 0); + append_stamped(&wal, 2, 1, 1); appender().append_snapshot_installed(10).unwrap(); - appender() - .append_calvin_applied(VShardId::new(3), 4, 0) - .unwrap(); - appender() - .append_calvin_applied(VShardId::new(1), 2, 3) - .unwrap(); + append_stamped(&wal, 3, 4, 0); + append_stamped(&wal, 1, 2, 3); appender().append_snapshot_installed(20).unwrap(); - appender() - .append_calvin_applied(VShardId::new(2), 6, 0) - .unwrap(); + append_stamped(&wal, 2, 6, 0); wal.sync().unwrap(); let group_of = |vshard: u32| match vshard { 1 => Some(10), @@ -483,7 +413,7 @@ mod tests { assert_eq!(all.keys().copied().collect::>(), vec![1, 2, 3]); for (vshard, recovered) in &all { - let alone = scan_applied(&wal, *vshard, group_of(*vshard)).unwrap(); + let alone = recover_vshard(&wal, *vshard, group_of(*vshard)); assert_eq!(recovered, &alone, "vShard {vshard}"); } assert_eq!(all[&1].applied_tail, [(2, 3)].into_iter().collect()); @@ -495,20 +425,14 @@ mod tests { /// vShards only. #[test] fn a_snapshot_install_record_clears_only_its_group() { - use crate::types::VShardId; let dir = TempDir::new().unwrap(); let wal = open_wal(&dir); - let appender = || wal.appender(crate::wal::manager::NO_APPLY_KEY); - appender() - .append_calvin_applied(VShardId::new(1), 3, 0) - .unwrap(); - appender() - .append_calvin_applied(VShardId::new(2), 3, 1) - .unwrap(); - appender().append_snapshot_installed(8).unwrap(); - appender() - .append_calvin_applied(VShardId::new(1), 5, 2) + append_stamped(&wal, 1, 3, 0); + append_stamped(&wal, 2, 3, 1); + wal.appender(crate::wal::manager::NO_APPLY_KEY) + .append_snapshot_installed(8) .unwrap(); + append_stamped(&wal, 1, 5, 2); wal.sync().unwrap(); let group_of = |vshard: u32| if vshard == 1 { Some(8) } else { Some(9) }; let catalog_dir = TempDir::new().unwrap(); @@ -525,26 +449,21 @@ mod tests { /// it: the install replaced the storage those positions wrote. #[test] fn a_group_snapshot_install_drops_the_markers_before_it() { - use crate::types::VShardId; let dir = TempDir::new().unwrap(); let wal = open_wal(&dir); let appender = || wal.appender(crate::wal::manager::NO_APPLY_KEY); - appender() - .append_calvin_applied(VShardId::new(1), 3, 0) - .unwrap(); + append_stamped(&wal, 1, 3, 0); appender().append_snapshot_installed(8).unwrap(); - appender() - .append_calvin_applied(VShardId::new(1), 5, 2) - .unwrap(); + append_stamped(&wal, 1, 5, 2); appender().append_snapshot_installed(9).unwrap(); wal.sync().unwrap(); - let in_group = scan_applied(&wal, 1, Some(8)).unwrap(); + let in_group = recover_vshard(&wal, 1, Some(8)); assert_eq!(in_group.applied_tail, [(5, 2)].into_iter().collect()); assert_eq!(in_group.max_applied_epoch, 5); - let other_group = scan_applied(&wal, 1, Some(4)).unwrap(); - assert_eq!(other_group, read_applied_recovery(&wal, 1).unwrap()); + let other_group = recover_vshard(&wal, 1, Some(4)); + assert_eq!(other_group, recover_vshard(&wal, 1, None)); assert!(other_group.applied_tail.contains(&(3, 0))); } } diff --git a/nodedb/src/control/planner/calvin/dispatch.rs b/nodedb/src/control/planner/calvin/dispatch.rs index 71841cdc3..e5aeccc97 100644 --- a/nodedb/src/control/planner/calvin/dispatch.rs +++ b/nodedb/src/control/planner/calvin/dispatch.rs @@ -489,7 +489,7 @@ mod tests { // them (`session::commit::run_commit`). // // WHY this must stay `MultiShard`: only the `MultiShard` branch of COMMIT - // flushes through the Calvin barrier (`run_commit_calvin`), which validates + // commits through the Calvin barrier (`run_commit_calvin`), which validates // B's read slice on B's OWNING node using the real per-shard `read_lsn`. If a // foreign read failed to widen the class, COMMIT will take the `SingleShard` // branch and run only the local-WAL `si_conflict_abort`, which never sees a diff --git a/nodedb/src/control/planner/calvin/edge_sequencing.rs b/nodedb/src/control/planner/calvin/edge_sequencing.rs index bd1d5b23c..a2fb73234 100644 --- a/nodedb/src/control/planner/calvin/edge_sequencing.rs +++ b/nodedb/src/control/planner/calvin/edge_sequencing.rs @@ -8,10 +8,9 @@ //! version applied below it. So on every replica, and under any clock skew, //! a TRUNCATE hides exactly the versions sequenced before it. //! -//! A data-group Raft entry cannot give that. Each replica stamps it from its -//! own clock, and each replica applies it at its own point among the Calvin -//! flushes. It also takes no Calvin lock, so a node delete's guard does not -//! order against it. +//! A plain data-group Raft entry cannot give that. It carries no Calvin +//! ordinal, so its edge versions order against no TRUNCATE cut. It also +//! takes no Calvin lock, so a node delete's guard does not order against it. //! //! The Raft propose seams (`propose_replicated_entry`, `propose_sync_write`) //! hand every edge write here. A session COMMIT that buffered an edge write diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/graph.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/graph.rs index 4b04c1095..58c935872 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/graph.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/graph.rs @@ -61,13 +61,13 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &GraphOp) -> crate::Result<()> } => keys.node(collection.as_str(), node_id), // A TRUNCATE's edge share locks no edge: it records a cut at its // transaction's ordinal, which hides every version applied below it - // whatever order this vShard flushes the edge writes in. + // whatever order this vShard installs the edge writes in. GraphOp::TruncateEdges { collection, vshard } => keys.home(collection.as_str(), *vshard), // A CRDT delete's presence guard locks every document it names, // stored or not, by the key every writer of that document locks. Its // `Intent` on the collection orders it against every collection-wide // write. So it runs after every lower writer of those documents - // flushed. + // installed. GraphOp::NodePresenceGuard { collection, vshard, diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/plan.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/plan.rs index 8ed05a490..e1236fb40 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/plan.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/plan.rs @@ -201,7 +201,7 @@ mod tests { /// An upsert binds and stores `x` at a lower position. A delete planned /// while `x` was unbound names it absent and carries an unbound delete. - /// Its guard must run after the upsert flushed, see `x` stored, and + /// Its guard must run after the upsert installed, see `x` stored, and /// retry; before, it ran first and the delete was lost. #[test] fn a_crdt_delete_of_an_unbound_document_waits_for_its_upsert() { diff --git a/nodedb/src/control/planner/rls_injection/meta.rs b/nodedb/src/control/planner/rls_injection/meta.rs index 2d7f90e7b..dad359032 100644 --- a/nodedb/src/control/planner/rls_injection/meta.rs +++ b/nodedb/src/control/planner/rls_injection/meta.rs @@ -106,7 +106,6 @@ pub(super) fn inject_meta(ctx: &RlsCtx<'_>, op: &mut MetaOp) -> crate::Result<() | MetaOp::DropTxnOverlay { .. } | MetaOp::MarkSavepoint { .. } | MetaOp::RollbackToSavepoint { .. } - | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. } | MetaOp::CalvinResolve { .. } | MetaOp::ApplyTransactionRedo { .. } diff --git a/nodedb/src/control/planner/rls_injection/permission_tree/meta.rs b/nodedb/src/control/planner/rls_injection/permission_tree/meta.rs index 6136fb15c..116f32989 100644 --- a/nodedb/src/control/planner/rls_injection/permission_tree/meta.rs +++ b/nodedb/src/control/planner/rls_injection/permission_tree/meta.rs @@ -105,7 +105,6 @@ pub(super) fn apply_meta(ctx: &PermCtx<'_>, op: &mut MetaOp) -> crate::Result<() | MetaOp::DropTxnOverlay { .. } | MetaOp::MarkSavepoint { .. } | MetaOp::RollbackToSavepoint { .. } - | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. } | MetaOp::CalvinResolve { .. } | MetaOp::ApplyTransactionRedo { .. } diff --git a/nodedb/src/control/security/identity/plan_permission.rs b/nodedb/src/control/security/identity/plan_permission.rs index ae2467d47..945bcb725 100644 --- a/nodedb/src/control/security/identity/plan_permission.rs +++ b/nodedb/src/control/security/identity/plan_permission.rs @@ -394,7 +394,6 @@ pub fn required_permission(plan: &crate::bridge::envelope::PhysicalPlan) -> Perm MetaOp::CalvinExecuteStatic { .. } | MetaOp::CalvinExecutePassive { .. } | MetaOp::CalvinExecuteActive { .. } - | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. }, ) => Permission::Write, diff --git a/nodedb/src/control/server/dispatch_utils/change_events/publish.rs b/nodedb/src/control/server/dispatch_utils/change_events/publish.rs index 2b73cf57b..ff024bcd6 100644 --- a/nodedb/src/control/server/dispatch_utils/change_events/publish.rs +++ b/nodedb/src/control/server/dispatch_utils/change_events/publish.rs @@ -204,5 +204,5 @@ pub(crate) fn publish_settled_changes( shared.change_stream.settle_group(group_id, first, last); shared .change_stream - .forward(shared, ChangePartition::Group(group_id)); + .forward(shared, ChangePartition(group_id)); } diff --git a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/driver.rs b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/driver.rs index 4c7482709..7829bd0d3 100644 --- a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/driver.rs +++ b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/driver.rs @@ -196,10 +196,9 @@ pub(crate) async fn enqueue_write( }; // Write-admission gate: every write-class plan whose ordering is not already - // final passes here. On the Calvin route the deterministic scheduler applies - // the write, emits its own WriteEvents, and owns durability (the sequenced - // TxClass plus its own `CalvinApplied` WAL record), so no local WAL append or - // enqueue happens. A plain write with no RETURNING rows yields `None`, + // final passes here. On the Calvin route the deterministic scheduler owns + // the write: the sequenced TxClass, then the stamped redo entry the data + // group applies. No local WAL append or enqueue happens. A plain write with no RETURNING rows yields `None`, // synthesized into a bare `Ok`. let target = AdmissionTarget { tenant_id, diff --git a/nodedb/src/control/server/exchange/all_cores/dispatch.rs b/nodedb/src/control/server/exchange/all_cores/dispatch.rs index a7db1a2a5..0924e0c0e 100644 --- a/nodedb/src/control/server/exchange/all_cores/dispatch.rs +++ b/nodedb/src/control/server/exchange/all_cores/dispatch.rs @@ -195,7 +195,6 @@ pub(crate) async fn execute_plan_all_local_cores( | MetaOp::RebuildIndex { .. } | MetaOp::PutSynonymGroup { .. } | MetaOp::DeleteSynonymGroup { .. } - | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. } => { generic_gather(state, tenant_id, database_id, plan, trace_id, txn_id).await } diff --git a/nodedb/src/control/server/http/routes/health.rs b/nodedb/src/control/server/http/routes/health.rs index 8fdfb9c35..85519e9e8 100644 --- a/nodedb/src/control/server/http/routes/health.rs +++ b/nodedb/src/control/server/http/routes/health.rs @@ -431,9 +431,9 @@ mod tests { vshard_id: 12, epoch: 40, position: 3, - reason: "flush_failed", - step: "flush", - error: "CalvinFlush returned Error".to_string(), + reason: "redo_apply_failed", + step: "redo_apply", + error: "the stamped redo install returned Error".to_string(), }); let (status, body) = healthz_body(state).await; @@ -444,8 +444,8 @@ mod tests { assert_eq!(body["vshard_id"], 12); assert_eq!(body["epoch"], 40); assert_eq!(body["position"], 3); - assert_eq!(body["halt_reason"], "flush_failed"); - assert_eq!(body["step"], "flush"); + assert_eq!(body["halt_reason"], "redo_apply_failed"); + assert_eq!(body["step"], "redo_apply"); } #[tokio::test] diff --git a/nodedb/src/control/server/shared/returning/inject.rs b/nodedb/src/control/server/shared/returning/inject.rs index 9f1214ed4..2ce734d75 100644 --- a/nodedb/src/control/server/shared/returning/inject.rs +++ b/nodedb/src/control/server/shared/returning/inject.rs @@ -413,7 +413,6 @@ pub fn inject_returning_spec(plan: &mut PhysicalPlan, spec: ReturningSpec) { | MetaOp::DropTxnOverlay { .. } | MetaOp::MarkSavepoint { .. } | MetaOp::RollbackToSavepoint { .. } - | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. } | MetaOp::ResolveTxn { .. } | MetaOp::CalvinResolve { .. } diff --git a/nodedb/src/control/server/shared/session/commit_calvin.rs b/nodedb/src/control/server/shared/session/commit_calvin.rs index bbeec076f..c6a191101 100644 --- a/nodedb/src/control/server/shared/session/commit_calvin.rs +++ b/nodedb/src/control/server/shared/session/commit_calvin.rs @@ -160,9 +160,10 @@ async fn submit_commit_calvin( // independent single-vShard Calvin transaction, sequenced through the // SAME deterministic funnel the contended point-write path uses // (`build_single_vshard_tx_class` + `submit_calvin_routed`): the - // scheduler resolves it into a `TransactionRedo`, WAL-appends it, and - // the Calvin flush sets `epoch_system_ms` so every engine's bitemporal - // stamp is epoch-anchored and byte-identical on replay. + // scheduler resolves it into a `TransactionRedo` and proposes it as a + // stamped data-group entry. The install sets `epoch_system_ms`, so + // every engine's bitemporal stamp is epoch-anchored and + // byte-identical on replay. // // Non-atomic ACROSS vShards is preserved by construction: each group // is a separate submit-and-await with its own single-participant diff --git a/nodedb/src/control/server/shared/session/live.rs b/nodedb/src/control/server/shared/session/live.rs index 87da175b9..66c1c2c18 100644 --- a/nodedb/src/control/server/shared/session/live.rs +++ b/nodedb/src/control/server/shared/session/live.rs @@ -472,7 +472,7 @@ mod tests { let mut live = LiveSubscription::new("live_orders".into(), subscription); let event = |index, floor| { SequencedChangeEvent::new( - ChangePartition::Group(2), + ChangePartition(2), CdcOffset::data_event(0, index, 1), floor, DatabaseId::DEFAULT, diff --git a/nodedb/src/control/server/shared/write_admission/admission_keys.rs b/nodedb/src/control/server/shared/write_admission/admission_keys.rs index 7d3b951fb..d4d00001e 100644 --- a/nodedb/src/control/server/shared/write_admission/admission_keys.rs +++ b/nodedb/src/control/server/shared/write_admission/admission_keys.rs @@ -104,7 +104,6 @@ pub(super) fn is_calvin_apply(plan: &PhysicalPlan) -> bool { MetaOp::CalvinExecuteStatic { .. } | MetaOp::CalvinExecutePassive { .. } | MetaOp::CalvinExecuteActive { .. } - | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. } | MetaOp::CalvinResolve { .. } ) diff --git a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs index 70fd4a5fc..9a085dae9 100644 --- a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs +++ b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs @@ -359,7 +359,6 @@ pub fn plan_requires_txn_buffering(plan: &PhysicalPlan) -> bool { | MetaOp::DropTxnOverlay { .. } | MetaOp::MarkSavepoint { .. } | MetaOp::RollbackToSavepoint { .. } - | MetaOp::CalvinFlush { .. } | MetaOp::CalvinDrop { .. } | MetaOp::CalvinResolve { .. } | MetaOp::ResolveTxn { .. } @@ -1971,13 +1970,6 @@ mod tests { txn_id: TxnId::new(1), savepoint: 1, }), - PhysicalPlan::Meta(MetaOp::CalvinFlush { - epoch: 0, - position: 0, - redo: Vec::new(), - collections: Vec::new(), - sum_targets: Vec::new(), - }), PhysicalPlan::Meta(MetaOp::CalvinDrop { epoch: 0, position: 0, diff --git a/nodedb/src/control/server/shared/write_admission/route.rs b/nodedb/src/control/server/shared/write_admission/route.rs index ae5892b0c..cb3806279 100644 --- a/nodedb/src/control/server/shared/write_admission/route.rs +++ b/nodedb/src/control/server/shared/write_admission/route.rs @@ -124,125 +124,12 @@ pub fn route_write_to_calvin<'a>( // vshard must be built with the single-vshard opt-in — it sequences // through the scheduler to serialize on the SAME shared per-vShard // `LockManager` the holder is on, rather than being rejected as a - // (spuriously) single-vshard multi-shard dispatch. - // A timeseries ingest resolves to its rows here, before it is - // sequenced: every replica resolves a sequenced write on its own. - let resolved = crate::control::write_resolve::resolve_tasks_for_log( - shared, - std::slice::from_ref(&task), - ) - .await?; - let tasks = resolved.unwrap_or_else(|| vec![task]); - // The submitter's resolve knows the rows the ingest stores and the - // lines it rejected. The client reads both from this answer, not - // from the scheduler's. - let counts = match tasks.first() { - Some(task) => crate::control::write_resolve::resolved_ingest_counts(&task.plan)?, - None => None, - }; - let mut tx_class = build_single_vshard_tx_class(&tasks, tenant_id, &[])?; + // (spuriously) single-vshard multi-shard dispatch. The gate never + // routes an append (timeseries ingest, columnar insert), so no + // unresolved ingest reaches the builder. + let mut tx_class = + build_single_vshard_tx_class(std::slice::from_ref(&task), tenant_id, &[])?; tx_class.set_event_source(event_source.wal_code()); - let applied = submit_calvin_routed(shared, tx_class).await?; - Ok(with_ingest_counts(applied, counts)) + submit_calvin_routed(shared, tx_class).await }) } - -/// `applied`, carrying the resolved ingest's counts when there are any. The -/// apply's counts win: its install rejected the rows that conflict with the -/// schema at its position. The resolve's `counts` answer only when the -/// apply's answer carries none. An error answer stays as the scheduler gave -/// it. -fn with_ingest_counts(applied: Option, counts: Option>) -> Option { - let Some(counts) = counts else { - return applied; - }; - match applied { - Some(response) if response.status != crate::bridge::envelope::Status::Ok => Some(response), - Some(response) - if crate::control::write_resolve::carries_applied_ingest_counts( - response.payload.as_bytes(), - ) => - { - Some(response) - } - Some(response) => Some(Response { - payload: crate::bridge::envelope::Payload::from_vec(counts), - ..response - }), - None => { - let mut response = bare_ok_response(RequestId::new(0)); - response.payload = crate::bridge::envelope::Payload::from_vec(counts); - Some(response) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn counts(accepted: u64, rejected: u64) -> Vec { - nodedb_types::json_to_msgpack(&serde_json::json!({ - "accepted": accepted, - "rejected": rejected, - "collection": "metrics", - })) - .expect("encode counts") - } - - fn decoded(response: &Response) -> serde_json::Value { - let json = crate::data::executor::response_codec::decode_payload_to_json( - response.payload.as_bytes(), - ); - sonic_rs::from_str(&json).expect("decode counts") - } - - /// A Calvin-routed timeseries ingest answers the rejected count its - /// submitter's resolve found, whether or not the scheduler answered. - #[test] - fn a_routed_ingest_answers_its_resolved_counts() { - let bare = with_ingest_counts(None, Some(counts(2, 1))).expect("an answer"); - assert_eq!(decoded(&bare)["rejected"], serde_json::json!(1)); - - let applied = with_ingest_counts( - Some(bare_ok_response(RequestId::new(3))), - Some(counts(2, 1)), - ) - .expect("an answer"); - assert_eq!(applied.request_id, RequestId::new(3)); - assert_eq!(decoded(&applied)["accepted"], serde_json::json!(2)); - assert_eq!(decoded(&applied)["rejected"], serde_json::json!(1)); - } - - /// The Calvin completion hands back the flush's answer. When it carries - /// the install's counts, the client gets the apply's count: here the - /// install rejected one row the resolve accepted. - #[test] - fn a_routed_ingest_answers_the_apply_counts_when_the_flush_carries_them() { - use crate::engine::timeseries::install_counts::{TsInstallCount, TsInstallCounts}; - let mut flushed = bare_ok_response(RequestId::new(4)); - flushed.payload = crate::bridge::envelope::Payload::from_vec( - TsInstallCounts::new(vec![TsInstallCount { - collection: "metrics".into(), - accepted: 1, - rejected: 2, - }]) - .to_bytes() - .expect("encode install counts"), - ); - let applied = with_ingest_counts(Some(flushed), Some(counts(2, 1))).expect("an answer"); - assert_eq!(decoded(&applied)["accepted"], serde_json::json!(1)); - assert_eq!(decoded(&applied)["rejected"], serde_json::json!(2)); - assert_eq!( - decoded(&applied)["collection"], - serde_json::json!("metrics") - ); - } - - /// A routed write that is not a resolved ingest keeps the scheduler's - /// answer. - #[test] - fn a_routed_non_ingest_keeps_the_scheduler_answer() { - assert!(with_ingest_counts(None, None).is_none()); - } -} diff --git a/nodedb/src/control/state/calvin_ack_result.rs b/nodedb/src/control/state/calvin_ack_result.rs index d50b74bec..980484068 100644 --- a/nodedb/src/control/state/calvin_ack_result.rs +++ b/nodedb/src/control/state/calvin_ack_result.rs @@ -23,7 +23,7 @@ pub const ACK_ROWS_FRAME_BUDGET: u64 = (nodedb_cluster::rpc_codec::MAX_RPC_PAYLO /// The `RETURNING` answer of a participant's slice. #[derive(Debug, Clone, PartialEq, Eq, zerompk::ToMessagePack, zerompk::FromMessagePack)] pub enum AckReturning { - /// The encoded rows, as the flush answered them. + /// The encoded rows, as the install answered them. Rows { rows: Vec }, /// The rows took `bytes`, past `limit`. OverLimit { bytes: u64, limit: u64 }, @@ -52,7 +52,7 @@ pub struct CalvinAckResult { /// The plain answer of a primary-write slice. #[derive(Debug, Clone, PartialEq, Eq, zerompk::ToMessagePack, zerompk::FromMessagePack)] pub enum AckReply { - /// The encoded answer, as the flush answered it. + /// The encoded answer, as the install answered it. Payload { payload: Vec }, /// The install stored the writes, and rendering the answer failed. Failed { detail: String }, @@ -68,7 +68,7 @@ pub struct AckSlice { } impl CalvinAckResult { - /// The report of a flush that answered `response` for `slice`. `limit` + /// The report of an install that answered `response` for `slice`. `limit` /// bounds the rows it carries. /// /// A primary-write slice always reports its answer: its rows, or its diff --git a/nodedb/src/control/state/calvin_local.rs b/nodedb/src/control/state/calvin_local.rs index 39dd4588a..e9808ec21 100644 --- a/nodedb/src/control/state/calvin_local.rs +++ b/nodedb/src/control/state/calvin_local.rs @@ -32,7 +32,7 @@ pub struct CalvinLocalState { /// session reads. 0 until a scheduler applies an epoch. pub last_applied_epoch: Arc, /// Node-global Calvin observability counters (write versions recorded, - /// read-set validation failures, commits flushed/dropped). + /// read-set validation failures, commits installed/dropped). pub counters: CalvinCounters, /// Local, in-process sidecar carrying the applied Data-Plane /// [`Response`](crate::bridge::envelope::Response) (affected-count and any diff --git a/nodedb/src/control/state/tenant_marks.rs b/nodedb/src/control/state/tenant_marks.rs index a83a95d6c..794208dd8 100644 --- a/nodedb/src/control/state/tenant_marks.rs +++ b/nodedb/src/control/state/tenant_marks.rs @@ -29,10 +29,9 @@ pub type GroupMarkEntry = (u64, u64, u8, String, u64); /// The apply path that recorded a mark. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MarkSite { - /// A committed data-group entry. + /// A committed data-group entry, a committed Calvin slice's install + /// included. ReplicatedApply, - /// A committed Calvin transaction's install. - CalvinFlush, /// A write a RESTORE re-issued. Restore, } @@ -42,7 +41,6 @@ impl MarkSite { pub fn as_str(self) -> &'static str { match self { Self::ReplicatedApply => "replicated apply", - Self::CalvinFlush => "calvin flush", Self::Restore => "restore re-issue", } } @@ -51,7 +49,6 @@ impl MarkSite { pub fn code(self) -> u8 { match self { Self::ReplicatedApply => 0, - Self::CalvinFlush => 1, Self::Restore => RESTORE_SITE_CODE, } } @@ -59,7 +56,6 @@ impl MarkSite { /// The site a persisted or wire code names. pub fn from_code(code: u8) -> Self { match code { - 1 => Self::CalvinFlush, RESTORE_SITE_CODE => Self::Restore, _ => Self::ReplicatedApply, } @@ -293,7 +289,7 @@ mod tests { fn a_mark_only_rises() { let marks = TenantMarks::default(); marks.raise(1, 7, 100, MarkSite::ReplicatedApply, Some("docs")); - marks.raise(1, 7, 50, MarkSite::CalvinFlush, None); + marks.raise(1, 7, 50, MarkSite::ReplicatedApply, None); let mark = marks.get(1, 7).expect("mark"); assert_eq!(mark.hlc, 100); assert_eq!(mark.collection.as_deref(), Some("docs")); @@ -339,7 +335,7 @@ mod tests { let dir = tempfile::tempdir().expect("tempdir"); let catalog = SystemCatalog::open(&dir.path().join("system.redb")).expect("catalog"); let marks = TenantMarks::default(); - marks.raise(3, 1, 900, MarkSite::CalvinFlush, Some("orders")); + marks.raise(3, 1, 900, MarkSite::ReplicatedApply, Some("orders")); marks.raise_restore(4, 1, 950, Some("orders"), 5); marks.persist(&catalog).expect("persist"); @@ -348,7 +344,7 @@ mod tests { reloaded.get(3, 1), Some(GroupMark { hlc: 900, - site: MarkSite::CalvinFlush, + site: MarkSite::ReplicatedApply, collection: Some("orders".to_owned()), restore_id: 0, }) diff --git a/nodedb/src/control/wal_replication/transaction_redo/apply.rs b/nodedb/src/control/wal_replication/transaction_redo/apply.rs index cab003697..9b90502f0 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/apply.rs +++ b/nodedb/src/control/wal_replication/transaction_redo/apply.rs @@ -87,7 +87,6 @@ pub(crate) async fn enqueue_transaction_redo( partition: target.vshard_id.as_u32(), epoch: position.epoch, index: position.log_index, - base: 0, }, ); stamped.apply_plan()? diff --git a/nodedb/src/control/write_resolve/mod.rs b/nodedb/src/control/write_resolve/mod.rs index 040817136..4a7f34f34 100644 --- a/nodedb/src/control/write_resolve/mod.rs +++ b/nodedb/src/control/write_resolve/mod.rs @@ -27,7 +27,7 @@ pub use run::{MAX_WRITE_RESOLVE_RETRIES, run_authorized_write_resolve, run_write pub use select::resolver_for_plan; pub use timeseries::TimeseriesWriteResolver; pub(crate) use timeseries::{ - carries_applied_ingest_counts, is_unresolved_ingest, rejected_lines, resolve_for_log, - resolve_ingest_plan, resolve_tasks_for_log, resolved_ingest_counts, + is_unresolved_ingest, rejected_lines, resolve_for_log, resolve_ingest_plan, + resolve_tasks_for_log, }; pub use vector::VectorWriteResolver; diff --git a/nodedb/src/control/write_resolve/timeseries.rs b/nodedb/src/control/write_resolve/timeseries.rs index 06be8d213..3ee4104ed 100644 --- a/nodedb/src/control/write_resolve/timeseries.rs +++ b/nodedb/src/control/write_resolve/timeseries.rs @@ -14,7 +14,6 @@ use nodedb_types::RlsWriteCheck; use crate::bridge::envelope::{PhysicalPlan, Status}; use crate::control::maintenance::clone_materializer::dispatch_local; use crate::control::state::SharedState; -use crate::engine::timeseries::install_counts::TsInstallCounts; use crate::engine::timeseries::resolved_ingest::{ RESOLVED_INGEST_FORMAT, ResolveBase, ResolvedTsBatch, TsDriftPolicy, }; @@ -247,44 +246,6 @@ pub(crate) fn rejected_lines(plan: &PhysicalPlan) -> crate::Result { } } -/// The count payload of the resolved timeseries ingest `plan`, in the shape -/// an install answers: the rows it stores and the lines its resolve -/// rejected. `None` for every other plan, and for an ingest with -/// `RETURNING`, which answers rows. -pub(crate) fn resolved_ingest_counts(plan: &PhysicalPlan) -> crate::Result>> { - let PhysicalPlan::Timeseries(TimeseriesOp::Ingest { - collection, - payload, - format, - returning: None, - .. - }) = plan - else { - return Ok(None); - }; - if format != RESOLVED_INGEST_FORMAT { - return Ok(None); - } - let batch = ResolvedTsBatch::from_bytes(payload)?; - let counts = serde_json::json!({ - "accepted": batch.rows.len(), - "rejected": batch.rejected, - "collection": collection.as_str(), - }); - nodedb_types::json_to_msgpack(&counts) - .map(Some) - .map_err(|e| crate::Error::Codec { - detail: format!("timeseries ingest counts: {e}"), - }) -} - -/// Whether an applied answer's `payload` carries its timeseries installs' -/// counts. Such a payload has the shape an install answers: the rows the -/// apply stored, and the lines and rows it rejected. -pub(crate) fn carries_applied_ingest_counts(payload: &[u8]) -> bool { - TsInstallCounts::from_payload(payload).is_some() -} - /// Whether `plan` is a timeseries ingest that has not resolved its rows. pub(crate) fn is_unresolved_ingest(plan: &PhysicalPlan) -> bool { matches!( diff --git a/nodedb/src/data/executor/core_loop/state.rs b/nodedb/src/data/executor/core_loop/state.rs index 19fefc4e2..fde56a685 100644 --- a/nodedb/src/data/executor/core_loop/state.rs +++ b/nodedb/src/data/executor/core_loop/state.rs @@ -429,7 +429,7 @@ pub struct CoreLoop { /// /// Set to `Some(ms)` by `execute_calvin_execute_static`, /// `execute_calvin_execute_active` and `execute_calvin_resolve` while they - /// stage or resolve a transaction's plans, and by `execute_calvin_flush` + /// stage or resolve a transaction's plans, and by `install_calvin_redo` /// while it renders the reply, then restored immediately after. Engine handlers that need "current time" (bitemporal sys_from, /// KV TTL expire_at, timeseries system_ms) call /// `self.epoch_system_ms.unwrap_or_else()` so that diff --git a/nodedb/src/data/executor/dispatch/meta.rs b/nodedb/src/data/executor/dispatch/meta.rs index 89e101470..ae3657c19 100644 --- a/nodedb/src/data/executor/dispatch/meta.rs +++ b/nodedb/src/data/executor/dispatch/meta.rs @@ -23,7 +23,7 @@ impl CoreLoop { MetaOp::Cancel { target_request_id } => self.execute_cancel(task, *target_request_id), // A committed transaction installs only through its redo record - // (`ApplyTransactionRedo`, `CalvinFlush`). A plan batch carries no + // (`ApplyTransactionRedo`). A plan batch carries no // record restart replay reads, so an Origin core refuses it. MetaOp::TransactionBatch { .. } => self.response_error( task, @@ -232,23 +232,6 @@ impl CoreLoop { self.execute_delete_synonym_group(task, *tenant_id, name) } - MetaOp::CalvinFlush { - epoch, - position, - redo, - collections, - sum_targets, - } => self.execute_calvin_flush( - task, - crate::data::executor::handlers::control::calvin::CalvinFlushRedo { - epoch: *epoch, - position: *position, - redo, - collections, - sum_targets, - }, - ), - MetaOp::CalvinDrop { epoch, position } => { self.execute_calvin_drop(task, *epoch, *position) } diff --git a/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs b/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs index d117deee4..b972645e6 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs @@ -80,14 +80,14 @@ impl CoreLoop { /// mutation and fires NO side effects. It stages each write plan into /// `txn_overlays` under the synthetic `TxnId` and buffers the plans in /// `commit_pending`, so a subsequent `CalvinResolve` builds one redo record - /// and [`CoreLoop::execute_calvin_flush`] installs it. + /// and [`CoreLoop::install_calvin_redo`] installs it. /// /// The one divergence from the static path: OLLP predicate verification /// (leader-only) runs HERE, before staging, via /// [`CoreLoop::verify_calvin_active_ollp`]. The dependent-read path has no /// LSN-versioned read-set to vote on; its conflict detector is the OLLP /// `actual != predicted` re-check. A mismatch returns `OllpRetryRequired` - /// and stages nothing, so no redo record is appended for it. The Control + /// and stages nothing, so no redo entry exists for it. The Control /// Plane scheduler releases locks and re-recons on `OllpRetryRequired`. /// /// `injected_reads` is retained on the wire for future plan variants that @@ -222,7 +222,7 @@ mod tests { /// The dependent-read ACTIVE path STAGES its writes (into `commit_pending` + /// the synthetic overlay) instead of applying them to base directly, so a /// Calvin-committed dependent-read write reaches the WAL as a redo record. - /// Staging routes it through the same resolve → redo → flush the static + /// Staging routes it through the same resolve → redo → install the static /// path uses. #[test] fn calvin_execute_active_stages_point_insert_into_overlay() { @@ -245,13 +245,13 @@ mod tests { let vshard_id = task.request.vshard_id.as_u32(); - // STAGED, not applied: the plans are buffered for the flush replay. + // STAGED, not applied: the plans are buffered for the resolve. assert!( core.calvin.commit_pending.contains_key(&(1, 0, vshard_id)), "active-path write must be STAGED into commit_pending, not applied directly" ); - // No base mutation at stage time — the row appears only after flush. + // No base mutation at stage time — the row appears only after install. let doc_id = nodedb_types::StorageKey::for_surrogate(Surrogate::new(7)); assert!( core.sparse @@ -279,7 +279,7 @@ mod tests { /// predicted set no longer matches live state returns `OllpRetryRequired` /// BEFORE staging anything — so no stale redo is WAL-appended and the /// coordinator can re-recon under a fresh attempt. Verifying at stage time - /// (not flush) is the one divergence from the static path. + /// (not install) is the one divergence from the static path. #[test] fn calvin_execute_active_ollp_drift_returns_retry_and_stages_nothing() { let dir = tempfile::tempdir().unwrap(); diff --git a/nodedb/src/data/executor/handlers/control/calvin/flush.rs b/nodedb/src/data/executor/handlers/control/calvin/flush.rs deleted file mode 100644 index de4b8fc23..000000000 --- a/nodedb/src/data/executor/handlers/control/calvin/flush.rs +++ /dev/null @@ -1,400 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Flush a staged Calvin transaction: install its committed redo record. - -use nodedb_physical::physical_plan::{MetaOp, PhysicalPlan, RedoOrigin, RedoSumTargets}; -use tracing::{debug, info_span}; - -use crate::bridge::envelope::{Payload, Response, Status}; -use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::handlers::transaction::redo_apply::CommittedRedo; -use crate::data::executor::task::ExecutionTask; - -/// The `MetaOp::CalvinFlush` fields the flush reads. -pub(in crate::data::executor) struct CalvinFlushRedo<'a> { - pub epoch: u64, - pub position: u32, - /// The encoded `RedoRecord` the scheduler appended at the request's LSN. - /// Empty when the transaction wrote nothing on this vShard. - pub redo: &'a [u8], - /// Every collection the local slice of the transaction writes. - pub collections: &'a [String], - /// The materialized-sum targets the local slice folds into. - pub sum_targets: &'a [RedoSumTargets], -} - -impl CoreLoop { - /// Flush the Calvin transaction staged under `(epoch, position)`. - /// - /// The flush installs the transaction's committed redo record through - /// [`CoreLoop::install_committed_redo_into`], the install every committed - /// transaction and restart replay run: validate, install with undo, then - /// settle and cover. An absent staged entry means a duplicate dispatch, - /// which answers `Ok` and installs nothing. - /// - /// The staged entry and its overlay stay until the install succeeds. A - /// refused install rolled every write back, so the scheduler can send the - /// same flush again. - /// - /// The reply reads at the epoch instant, so every replica sees the same - /// live rows. The response carries the reply the plans decided when they - /// staged. A document or CRDT `RETURNING` row is read from base after the - /// install, so it reports the row the install stored. A reply that fails - /// to render leaves the flush `Ok`, since the record installed. Its error - /// travels in `error_code` for the statement to report. - pub(in crate::data::executor) fn execute_calvin_flush( - &mut self, - task: &ExecutionTask, - flush: CalvinFlushRedo<'_>, - ) -> Response { - let CalvinFlushRedo { - epoch, - position, - redo, - collections, - sum_targets, - } = flush; - let vshard_id = task.request.vshard_id.as_u32(); - let key = (epoch, position, vshard_id); - let Some((tenant_id, epoch_system_ms, origin)) = - self.calvin.commit_pending.get(&key).map(|pending| { - ( - pending.tenant_id, - pending.epoch_system_ms, - redo_origin(&pending.plans), - ) - }) - else { - debug!( - core = self.core_id, - epoch, position, vshard_id, "calvin flush: no staged commit (already resolved)" - ); - return self.response_ok(task); - }; - let _apply_span = info_span!( - "executor_apply", - epoch, - position, - vshard = vshard_id, - tenant_id = tenant_id.as_u64(), - trace_id = ?task.request.trace_id, - ) - .entered(); - // The record's graph versions stamp at least the transaction's - // ordinal, so a later local write stamps above them. - let txn_ordinal = nodedb_types::calvin_txn_ordinal(epoch_system_ms, position); - self.hlc.update_from_remote(txn_ordinal); - - // What each resolved timeseries batch of the record stored: the - // reply answers the apply's counts and stored rows, not the resolve's. - let mut ts_installs = Vec::new(); - let mut response = if redo.is_empty() { - self.response_ok(task) - } else { - self.install_committed_redo_into( - task, - tenant_id.as_u64(), - CommittedRedo { - redo, - collections, - sum_targets, - }, - origin, - &mut ts_installs, - ) - }; - if response.status != Status::Ok { - return response; - } - // Restart replay folds the record's sum targets from its stamp, so the - // folded target rows stay out of the record's group. - response.write_set.clear(); - // The record installed: the staged entry and its overlay are spent. - let reply = self - .calvin - .commit_pending - .remove(&key) - .map(|pending| pending.reply) - .unwrap_or_default(); - self.drop_calvin_synthetic_overlay(epoch, position, vshard_id); - let prev_epoch_ms = self.epoch_system_ms; - self.epoch_system_ms = Some(epoch_system_ms); - let rendered = self.calvin_reply_payload(task, tenant_id.as_u64(), reply, &ts_installs); - self.epoch_system_ms = prev_epoch_ms; - match rendered { - Ok(payload) => response.payload = Payload::from_vec(payload), - Err(error) => response.error_code = Some(Box::new(error)), - } - response - } -} - -/// The origin of a Calvin transaction's redo record: a RESTORE when its -/// local plans install a RESTORE batch, a commit otherwise. A RESTORE's rows -/// passed their collection's rules when they were first written, so the -/// install runs only the checks a RESTORE runs. -fn redo_origin(plans: &[PhysicalPlan]) -> RedoOrigin { - if plans - .iter() - .any(|plan| matches!(plan, PhysicalPlan::Meta(MetaOp::RestoreRedo(_)))) - { - RedoOrigin::Restore - } else { - RedoOrigin::Commit - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::bridge::envelope::ErrorCode; - use crate::data::executor::core_loop::tests::make_core_with_dir; - use crate::data::executor::handlers::control::calvin_txn_id::calvin_synthetic_txn_id; - use crate::types::{DatabaseId, Lsn, TenantId}; - use nodedb_physical::physical_plan::{ - DocumentOp, PhysicalPlan, ReturningColumns, ReturningSpec, - }; - use nodedb_types::{QualifiedCollection, StorageKey, Surrogate}; - - use crate::engine::document::store::{CollectionConfig, IndexPath}; - - use super::super::shared::CalvinExecCtx; - use super::super::shared::test_support::{ - bulk_delete_plan, doc_value, make_task, point_insert_plan, seed_row, - }; - - const CTX: CalvinExecCtx = CalvinExecCtx { - epoch: 1, - position: 0, - epoch_system_ms: 0, - }; - - /// Stage `plans` at `(1, 0)`, resolve them, and return the redo bytes. - fn stage_and_resolve(core: &mut CoreLoop, plans: &[PhysicalPlan]) -> Vec { - let task = make_task(); - let staged = - core.execute_calvin_execute_static(&task, CTX, &TenantId::new(1), plans, &[], &[]); - assert_eq!(staged.status, Status::Ok, "{:?}", staged.error_code); - let resolved = core.execute_calvin_resolve(&task, 1, 0); - assert_eq!(resolved.status, Status::Ok, "{:?}", resolved.error_code); - let answer: nodedb_physical::physical_plan::CalvinResolved = - zerompk::from_msgpack(resolved.payload.as_bytes()).expect("decode resolved answer"); - answer.redo - } - - fn flush_at(core: &mut CoreLoop, lsn: u64, redo: &[u8], collections: &[String]) -> Response { - let mut task = make_task(); - task.wal_lsn = Some(Lsn::new(lsn)); - core.execute_calvin_flush( - &task, - CalvinFlushRedo { - epoch: 1, - position: 0, - redo, - collections, - sum_targets: &[], - }, - ) - } - - fn base_row(core: &CoreLoop, collection: &str, surrogate: u32) -> Option> { - core.sparse - .get( - DatabaseId::DEFAULT.as_u64(), - 1, - collection, - &StorageKey::for_surrogate(Surrogate::new(surrogate)), - ) - .expect("read base row") - } - - #[test] - fn calvin_flush_drops_synthetic_overlay() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - let redo = stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - - let vshard_id = make_task().request.vshard_id.as_u32(); - let synthetic = calvin_synthetic_txn_id(1, 0, vshard_id).unwrap(); - assert!(core.txn_overlays.contains_key(&synthetic)); - - let flushed = flush_at(&mut core, 40, &redo, &["orders".to_string()]); - assert_eq!(flushed.status, Status::Ok, "{:?}", flushed.error_code); - - assert!( - !core.txn_overlays.contains_key(&synthetic), - "flush must drop the synthetic overlay entry alongside commit_pending" - ); - assert!( - base_row(&core, "orders", 7).is_some(), - "the flush installs the staged row from the redo record" - ); - } - - /// The flush installs the redo record, not the staged plans. A row that - /// joins the bulk delete's predicate after staging stays, and the - /// predicted row goes. - #[test] - fn a_flush_installs_the_resolved_record_not_a_replay_of_the_plans() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - seed_row(&mut core, "orders", 1); - let redo = stage_and_resolve(&mut core, &[bulk_delete_plan("orders", Some(vec![1]))]); - - seed_row(&mut core, "orders", 2); - let flushed = flush_at(&mut core, 41, &redo, &["orders".to_string()]); - - assert_eq!(flushed.status, Status::Ok, "{:?}", flushed.error_code); - assert!(base_row(&core, "orders", 1).is_none()); - assert!(base_row(&core, "orders", 2).is_some()); - } - - /// The flush answers with the reply the staged statement computed: - /// RETURNING rows for a point insert that asked for them. - #[test] - fn a_calvin_flush_answers_the_returning_rows_its_statement_staged() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - let plan = PhysicalPlan::Document(DocumentOp::PointInsert { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "orders"), - document_id: "o9".to_string(), - value: doc_value("a", "returned"), - if_absent: false, - surrogate: Surrogate::new(9), - returning: Some(ReturningSpec { - columns: ReturningColumns::Star, - }), - rls_filters: Vec::new(), - resolved_sum_targets: Vec::new(), - deferred_sum_targets: Vec::new(), - }); - let redo = stage_and_resolve(&mut core, &[plan]); - - let flushed = flush_at(&mut core, 42, &redo, &["orders".to_string()]); - - assert_eq!(flushed.status, Status::Ok, "{:?}", flushed.error_code); - let payload = flushed.payload.as_bytes(); - assert!( - payload.windows(b"returned".len()).any(|w| w == b"returned"), - "the flush reply carries the inserted row" - ); - assert!(base_row(&core, "orders", 9).is_some()); - } - - /// The install notes the index values of every document row at the - /// flush's LSN, so no flush-time staging carries them to a later op. - #[test] - fn a_calvin_flush_notes_index_values_at_the_redo_lsn() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - let mut config = CollectionConfig::new("orders"); - config.index_paths.push(IndexPath::new("a")); - core.doc_configs.insert( - (DatabaseId::DEFAULT, TenantId::new(1), "orders".to_string()), - config, - ); - let redo = stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - - let flushed = flush_at(&mut core, 43, &redo, &["orders".to_string()]); - - assert_eq!(flushed.status, Status::Ok, "{:?}", flushed.error_code); - assert_eq!( - core.write_index.index_values.value_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "orders", - "a", - "1" - ), - Some(Lsn::new(43)), - "the install records the row's index value at the redo LSN" - ); - } - - /// A flush whose record fails to install answers the install's error and - /// leaves nothing of the record in base. - #[test] - fn a_calvin_flush_whose_record_cannot_install_answers_an_error() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - - let flushed = flush_at(&mut core, 44, &[0xff, 0x00], &["orders".to_string()]); - - assert_eq!(flushed.status, Status::Error); - assert!(!matches!( - flushed.error_code.as_deref(), - Some(ErrorCode::OllpRetryRequired) - )); - assert!(base_row(&core, "orders", 7).is_none()); - } - - /// A refused install keeps the staged entry and its overlay, so the - /// scheduler's resent flush installs the record. - #[test] - fn a_refused_install_keeps_the_staged_commit_for_the_resent_flush() { - use crate::data::executor::handlers::transaction::redo_apply::test_commit::failing_install_sub_record; - use crate::wal::RedoRecord; - - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - let redo = stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - let mut failing = RedoRecord::from_bytes(&redo).expect("decode redo"); - failing.ops.push(failing_install_sub_record()); - let failing = failing.to_bytes().expect("encode redo"); - let vshard_id = make_task().request.vshard_id.as_u32(); - let synthetic = calvin_synthetic_txn_id(1, 0, vshard_id).unwrap(); - - let refused = flush_at(&mut core, 45, &failing, &["orders".to_string()]); - - assert!( - matches!( - refused.error_code.as_deref(), - Some(ErrorCode::RetryableRefusal { .. }) - ), - "{:?}", - refused.error_code - ); - assert!(core.calvin.commit_pending.contains_key(&(1, 0, vshard_id))); - assert!(core.txn_overlays.contains_key(&synthetic)); - assert!(base_row(&core, "orders", 7).is_none()); - - let flushed = flush_at(&mut core, 46, &redo, &["orders".to_string()]); - - assert_eq!(flushed.status, Status::Ok, "{:?}", flushed.error_code); - assert!(base_row(&core, "orders", 7).is_some()); - assert!(!core.calvin.commit_pending.contains_key(&(1, 0, vshard_id))); - assert!(!core.txn_overlays.contains_key(&synthetic)); - } - - /// A reply that fails to render after a successful install leaves the - /// flush `Ok` and carries the render error for the statement. - #[test] - fn a_reply_render_error_leaves_the_installed_flush_ok() { - use crate::data::executor::handlers::control::calvin_reply::CalvinReply; - - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - let redo = stage_and_resolve(&mut core, &[point_insert_plan("orders", "o1", 7)]); - let vshard_id = make_task().request.vshard_id.as_u32(); - core.calvin - .commit_pending - .get_mut(&(1, 0, vshard_id)) - .expect("staged commit") - .reply = CalvinReply::unrenderable_for_test("orders"); - - let flushed = flush_at(&mut core, 47, &redo, &["orders".to_string()]); - - assert_eq!(flushed.status, Status::Ok); - assert!( - matches!( - flushed.error_code.as_deref(), - Some(ErrorCode::Internal { .. }) - ), - "{:?}", - flushed.error_code - ); - assert!(base_row(&core, "orders", 7).is_some()); - assert!(!core.calvin.commit_pending.contains_key(&(1, 0, vshard_id))); - } -} diff --git a/nodedb/src/data/executor/handlers/control/calvin/mod.rs b/nodedb/src/data/executor/handlers/control/calvin/mod.rs index 42583c3cf..fcf783aa7 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/mod.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/mod.rs @@ -31,11 +31,11 @@ mod active_passive; mod discard; -mod flush; mod shared; mod static_stage; #[cfg(test)] mod test_commit; -pub(in crate::data::executor) use flush::CalvinFlushRedo; pub(in crate::data::executor) use shared::CalvinExecCtx; +#[cfg(test)] +pub(in crate::data::executor) use shared::test_support; diff --git a/nodedb/src/data/executor/handlers/control/calvin/shared.rs b/nodedb/src/data/executor/handlers/control/calvin/shared.rs index dc363a522..45448cf02 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/shared.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/shared.rs @@ -54,7 +54,7 @@ impl CoreLoop { } #[cfg(test)] -pub(in crate::data::executor::handlers::control::calvin) mod test_support { +pub(in crate::data::executor) mod test_support { use std::time::{Duration, Instant}; use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan, TimeseriesOp}; @@ -112,7 +112,7 @@ pub(in crate::data::executor::handlers::control::calvin) mod test_support { nodedb_types::value_to_msgpack(&Value::Object(obj)).unwrap() } - pub(in crate::data::executor::handlers::control::calvin) fn point_insert_plan( + pub(in crate::data::executor) fn point_insert_plan( collection: &str, document_id: &str, surrogate: u32, @@ -148,7 +148,7 @@ pub(in crate::data::executor::handlers::control::calvin) mod test_support { }) } - pub(in crate::data::executor::handlers::control::calvin) fn bulk_delete_plan( + pub(in crate::data::executor) fn bulk_delete_plan( collection: &str, predicted: Option>, ) -> PhysicalPlan { @@ -167,7 +167,7 @@ pub(in crate::data::executor::handlers::control::calvin) mod test_support { /// Seed a row directly into base storage (bypassing Calvin staging), the /// pre-existing state the active-path OLLP verifier scans against. - pub(in crate::data::executor::handlers::control::calvin) fn seed_row( + pub(in crate::data::executor) fn seed_row( core: &mut CoreLoop, collection: &str, surrogate: u32, diff --git a/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs b/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs index 2fa030878..706be5e9d 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs @@ -585,7 +585,7 @@ mod tests { )); assert!( !core.calvin.commit_pending.contains_key(&(21, 3, vshard)), - "a rejected stage cannot reach the TransactionRedo-producing flush path" + "a rejected stage cannot reach the TransactionRedo-producing resolve path" ); assert!(!core.txn_overlays.contains_key(&overflow_id)); } diff --git a/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs b/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs index bef35504f..bff0e14e4 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs @@ -1,9 +1,12 @@ // SPDX-License-Identifier: BUSL-1.1 //! Test driver for a committed Calvin transaction on one core: stage the -//! plans, resolve them into the redo record, and flush it. +//! plans, resolve them into the redo record, and install the record as the +//! slice's stamped `ApplyTransactionRedo`. -use nodedb_physical::physical_plan::{CalvinResolved, PhysicalPlan}; +use nodedb_physical::physical_plan::{ + CalvinInstall, CalvinResolved, MetaOp, PhysicalPlan, RedoOrigin, +}; use crate::bridge::envelope::{Response, Status}; use crate::control::wal_replication::transaction_redo::collections::written_collections; @@ -12,15 +15,14 @@ use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::task::ExecutionTask; use crate::types::{Lsn, TenantId}; -use super::flush::CalvinFlushRedo; use super::shared::CalvinExecCtx; impl CoreLoop { - /// Commit `plans` as the Calvin transaction at `(epoch, 0)` and flush its - /// redo record at `lsn`, the path a committed Calvin transaction takes on - /// its participant core. + /// Commit `plans` as the Calvin transaction at `(epoch, 0)` and install + /// its redo record at `lsn`, the path a committed Calvin slice takes on + /// every replica of its vShard. /// - /// Returns the first refusal: the stage, the resolve, or the flush. + /// Returns the first refusal: the stage, the resolve, or the install. pub(in crate::data::executor) fn calvin_commit_for_test( &mut self, task: &ExecutionTask, @@ -57,8 +59,8 @@ impl CoreLoop { if resolved.status != Status::Ok { return resolved; } - let redo = match zerompk::from_msgpack::(resolved.payload.as_bytes()) { - Ok(answer) => answer.redo, + let answer = match zerompk::from_msgpack::(resolved.payload.as_bytes()) { + Ok(answer) => answer, Err(error) => { return self.response_error( task, @@ -68,20 +70,22 @@ impl CoreLoop { ); } }; - let mut request = task.request.clone(); - request.wal_lsn = Some(Lsn::new(lsn)); - let flush_task = ExecutionTask::new(request); - let collections = written_collections(plans); - let sum_targets = redo_sum_targets(plans); - self.execute_calvin_flush( - &flush_task, - CalvinFlushRedo { + let install = PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { + redo: answer.redo, + collections: written_collections(plans), + sum_targets: redo_sum_targets(plans), + origin: RedoOrigin::Commit, + calvin: Some(CalvinInstall { epoch, position: 0, - redo: &redo, - collections: &collections, - sum_targets: &sum_targets, - }, - ) + epoch_system_ms, + reply: answer.reply, + user_write: true, + }), + }); + let mut request = task.request.clone(); + request.wal_lsn = Some(Lsn::new(lsn)); + let install_task = ExecutionTask::new(request); + self.execute_plan(&install_task, &install) } } diff --git a/nodedb/src/data/executor/handlers/control/calvin_active_verify.rs b/nodedb/src/data/executor/handlers/control/calvin_active_verify.rs index fa82109ba..0309aa5d6 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_active_verify.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_active_verify.rs @@ -2,7 +2,7 @@ //! Stage-time OLLP predicate verification for the dependent-read ACTIVE Calvin //! path, which carries no versioned read-set. The leader-only `actual != -//! predicted` re-check must run HERE, before staging — at flush time a +//! predicted` re-check must run HERE, before staging — at install time a //! mismatch is swallowed as a degraded shard instead. Mirrors the //! `BulkDelete`/`BulkUpdate` arms of [`CoreLoop::stage_calvin_overlay`]. diff --git a/nodedb/src/data/executor/handlers/control/calvin_overlay_stage.rs b/nodedb/src/data/executor/handlers/control/calvin_overlay_stage.rs index 7a7dcd93a..7b343c716 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_overlay_stage.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_overlay_stage.rs @@ -6,8 +6,8 @@ //! //! Staging is the producer side for `CalvinResolve`, which reads the overlay //! the same way `MetaOp::ResolveTxn` does for session transactions -//! (`resolve/entry.rs`). The redo record it builds is what the flush -//! installs. +//! (`resolve/entry.rs`). The redo record it builds is what the stamped redo +//! entry installs. use nodedb_physical::physical_plan::{ ArrayOp, ColumnarOp, CrdtOp, DocumentOp, GraphOp, PhysicalPlan, TimeseriesOp, VectorOp, @@ -288,7 +288,7 @@ impl CoreLoop { } // Every columnar write stages its post-image the way a session // write does, so the transaction's redo carries the rows the - // flush installs: an ON CONFLICT merge and a predicate DML's + // install writes: an ON CONFLICT merge and a predicate DML's // matched rows are resolved once, here, against the state this // position observes. PhysicalPlan::Columnar( @@ -395,7 +395,7 @@ impl CoreLoop { /// own parallel overlay, same as a session transaction's /// `execute_stage_graph`), and `array_txn_overlays` (ARRAY cell ops, /// same as `execute_stage_array`). Called from both - /// [`CoreLoop::execute_calvin_flush`] and [`CoreLoop::execute_calvin_drop`] + /// [`CoreLoop::install_calvin_redo`] and [`CoreLoop::execute_calvin_drop`] /// so neither overlay outlives the `commit_pending` entry it shadows. /// Idempotent: a missing key (already removed, or the id derivation /// itself failing) is a silent no-op — the same shape as the @@ -411,7 +411,7 @@ impl CoreLoop { ) { if let Ok(synthetic_txn_id) = calvin_synthetic_txn_id(epoch, position, vshard) { // The shared teardown also drops a columnar engine staging - // auto-created that the flush left empty. + // auto-created that the install left empty. self.drop_overlay_entry(synthetic_txn_id); } } diff --git a/nodedb/src/data/executor/handlers/control/calvin_overlay_stage_bulk.rs b/nodedb/src/data/executor/handlers/control/calvin_overlay_stage_bulk.rs index 8a05ca4cc..1bd0d93ff 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_overlay_stage_bulk.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_overlay_stage_bulk.rs @@ -11,8 +11,8 @@ //! live predicate rescan is NOT used as the row set, because a follower's //! local snapshot can legitimately lag the leader's verified prediction //! window. Re-deriving the row set locally would diverge across replicas. -//! The flush installs the redo record `CalvinResolve` builds from this -//! staging, so the staged rows are the rows the flush writes. The +//! The install applies the redo record `CalvinResolve` builds from this +//! staging, so the staged rows are the rows the install writes. The //! `stage_bulk_delete` / `stage_bulk_update` session-transaction handlers do //! a live rescan and are NOT reused here for this reason. //! @@ -92,7 +92,7 @@ pub(in crate::data::executor) struct CalvinBulkUpdateStage<'a> { impl CoreLoop { /// Stage a Calvin `BulkDelete` into the overlay: one tombstone per /// predicted surrogate, resolved to its doc-id via - /// `ollp_predicted_doc_ids` — the identical primitive the flush apply + /// `ollp_predicted_doc_ids` — the identical primitive the install /// uses to derive `apply_ids`. NOT a live predicate rescan. Returns the /// number of predicted rows that exist. pub(in crate::data::executor) fn stage_calvin_bulk_delete( @@ -181,7 +181,7 @@ impl CoreLoop { /// via the exact same per-row transform `execute_bulk_update` / /// `stage_point_update` use (`CoreLoop::stage_apply_update`), and stage /// the post-image as a `Put`. Row set = predicted surrogates, matching - /// the flush apply set exactly (and correctly excluding rows a + /// the install set exactly (and correctly excluding rows a /// same-transaction `INSERT` created after the predicted set was /// computed at recon). /// @@ -241,7 +241,7 @@ impl CoreLoop { declared_primary_key, )?; // Decide the staged post-image against the write policy: this is - // the row the Calvin flush will install. + // the row the stamped redo install writes. let identity = stored_row_identity( &new_body, strict_schema.as_ref(), diff --git a/nodedb/src/data/executor/handlers/control/calvin_reply/mod.rs b/nodedb/src/data/executor/handlers/control/calvin_reply/mod.rs index 6d35ff9f0..ce9c7742b 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_reply/mod.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_reply/mod.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 -mod flush_read; mod images; +mod payload; mod reply; mod spec; mod stage; diff --git a/nodedb/src/data/executor/handlers/control/calvin_reply/flush_read.rs b/nodedb/src/data/executor/handlers/control/calvin_reply/payload.rs similarity index 94% rename from nodedb/src/data/executor/handlers/control/calvin_reply/flush_read.rs rename to nodedb/src/data/executor/handlers/control/calvin_reply/payload.rs index ddb6a763d..9f0f408a3 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_reply/flush_read.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_reply/payload.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 -//! The payload a flushed Calvin transaction answers with. +//! The payload an installed Calvin slice answers with. use super::images::{RowLocation, StoredRow}; use super::reply::{CalvinReply, InstalledTimeseries}; @@ -14,8 +14,8 @@ use crate::engine::timeseries::install_counts::TsInstallCounts; use crate::util::rmpv_value::rmpv_to_value; impl CoreLoop { - /// The payload `reply` answers with once the flush has installed the - /// transaction. Post-images are read from base after the install, so each row is what + /// The payload `reply` answers with once the redo install has written + /// the slice. Post-images are read from base after the install, so each row is what /// the install stored. /// /// A row the plan wrote that base does not hold after the install is an @@ -93,7 +93,7 @@ fn installed_timeseries_rows( return Err(ErrorCode::Internal { detail: format!( "calvin RETURNING: the install of timeseries ingest {} into '{}' is absent \ - from the flushed record", + from the installed record", installed.ordinal, installed.collection ), }); diff --git a/nodedb/src/data/executor/handlers/control/calvin_reply/reply.rs b/nodedb/src/data/executor/handlers/control/calvin_reply/reply.rs index 443ce0d98..015c01769 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_reply/reply.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_reply/reply.rs @@ -1,14 +1,14 @@ // SPDX-License-Identifier: BUSL-1.1 -//! The reply a staged Calvin transaction answers with when its flush -//! installs it. +//! The reply a staged Calvin transaction answers with when its redo record +//! installs. use nodedb_physical::physical_plan::ReturningSpec; use nodedb_types::{RowIdentity, Surrogate}; use super::target::RowEngine; -/// The reply a Calvin transaction's flush answers with. +/// The reply a Calvin transaction's install answers with. /// /// The transaction answers with its last `RETURNING` plan's rows, or with /// its last plan's affected count when no plan carries `RETURNING`. A plan @@ -20,7 +20,7 @@ pub(in crate::data::executor) enum CalvinReply { Count(Vec), /// `RETURNING` rows, decided when the plan staged. Rows(Vec), - /// `RETURNING` rows the flush reads from base after the install, so each + /// `RETURNING` rows the install reads from base after it writes, so each /// row is exactly what a `SELECT` reads. PostImages(PostImages), /// `RETURNING` rows of a resolved timeseries ingest: the rows its install @@ -36,7 +36,7 @@ impl Default for CalvinReply { } impl CalvinReply { - /// A reply the flush cannot render: post-images of a columnar row, which + /// A reply the install cannot render: post-images of a columnar row, which /// base keys by no surrogate. #[cfg(test)] pub(in crate::data::executor) fn unrenderable_for_test(collection: &str) -> Self { @@ -85,7 +85,7 @@ pub(in crate::data::executor) struct CalvinStaging { pub(super) owned_edges: u64, } -/// The rows of a `RETURNING` plan the flush reads after the install. +/// The rows of a `RETURNING` plan the install reads after it writes. #[derive(Debug)] pub(in crate::data::executor) struct PostImages { pub(super) spec: ReturningSpec, diff --git a/nodedb/src/data/executor/handlers/control/calvin_reply/stage.rs b/nodedb/src/data/executor/handlers/control/calvin_reply/stage.rs index b823adaaf..70a2c45f4 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_reply/stage.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_reply/stage.rs @@ -2,8 +2,8 @@ //! Stage one Calvin plan and fold its reply into the transaction's reply. //! -//! A Calvin flush installs the transaction's redo record, and a redo record -//! carries no reply. Each plan therefore decides its reply when it stages. +//! A Calvin slice installs from its redo record, and a redo record carries no +//! reply. Each plan therefore decides its reply when it stages. //! //! - A plan without `RETURNING` answers its staging handler's reply: the //! affected count against BASE ∪ OVERLAY. @@ -12,8 +12,7 @@ //! taken from BASE ∪ OVERLAY, so a row an earlier plan of the same //! transaction wrote is reported as that plan left it. //! - A delete reports each removed row's image from before the plan. -//! - A document or CRDT write reports each row as the flush's install -//! stored it, read back from base after the install. The install can +//! - A document or CRDT write reports each row as the install stored it, read back from base after the install. The install can //! rewrite a body, as hash chaining does. //! - A KV, vector-primary or columnar write reports the rows it staged, //! which are the exact bytes the install stores. @@ -58,7 +57,7 @@ impl CoreLoop { ) -> Result<(), ErrorCode> { let tid = tenant_id.as_u64(); // Every timeseries ingest becomes one redo sub-record, in plan order, - // so its position among them names its install at the flush. + // so its position among them names its install. let ts_ordinal = staging.ts_ingests; if matches!(plan, PhysicalPlan::Timeseries(TimeseriesOp::Ingest { .. })) { staging.ts_ingests += 1; @@ -75,7 +74,7 @@ impl CoreLoop { let reply = &mut staging.reply; match returning_target(plan) { // A resolved ingest's install decides which rows land and how - // they read, so the flush renders them from its own install. + // they read, so the reply renders them from that install. Some(target) if is_resolved_ingest(plan) => { *reply = CalvinReply::InstalledTimeseries(InstalledTimeseries { spec: target.spec.clone(), @@ -491,7 +490,7 @@ mod tests { } /// Commit `plans` as one Calvin transaction on a fresh core prepared by - /// `setup`, and return the flush's reply. + /// `setup`, and return the install's reply. fn commit(plans: &[PhysicalPlan], setup: impl FnOnce(&mut CoreLoop)) -> Response { let dir = tempfile::tempdir().expect("tempdir"); let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); diff --git a/nodedb/src/data/executor/handlers/control/calvin_resolve.rs b/nodedb/src/data/executor/handlers/control/calvin_resolve.rs index c06ea8b37..991fcef34 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_resolve.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_resolve.rs @@ -34,7 +34,7 @@ impl CoreLoop { /// Errors (rather than silently dropping data or producing an empty /// record) when no `commit_pending` entry exists for /// `(epoch, position, vshard)` — the transaction was never staged (or was - /// already flushed/dropped), and there is nothing to resolve. + /// already installed/dropped), and there is nothing to resolve. /// /// `DocumentOp::BulkUpdate` / `BulkDelete` plans are staged into the /// overlay by `stage_calvin_overlay` (via the predicted-surrogate-set @@ -406,7 +406,7 @@ mod tests { // them -- if staging re-derived the row set via a live scan instead // of trusting the predicted set, the redo would carry 3 deletes, not // 2. This is the determinism proof: staging must key off the - // predicted set the flush applies, not a fresh predicate scan. + // predicted set the install applies, not a fresh predicate scan. seed_row(&mut core, "orders", 20, "a", "1"); seed_row(&mut core, "orders", 21, "a", "1"); seed_row(&mut core, "orders", 22, "a", "1"); diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest_resolved_outcome.rs b/nodedb/src/data/executor/handlers/timeseries/ingest_resolved_outcome.rs index 1e0c78413..e4d3d7c39 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest_resolved_outcome.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest_resolved_outcome.rs @@ -10,7 +10,7 @@ //! its durable outcome (`install_outcome`) before the first row lands, so //! WAL catch-up rebuilds exactly the events the install emits. //! - A committed redo install records its counts and landed images in the -//! open apply scope, so the apply answers the counts and a Calvin flush +//! open apply scope, so the apply answers the counts and a Calvin install //! renders its `RETURNING` rows from the stored rows. use crate::data::executor::core_loop::CoreLoop; diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs index a1b990efe..1f464607b 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs @@ -10,13 +10,20 @@ use nodedb_physical::physical_plan::{ use nodedb_types::{QualifiedCollection, StorageKey, Surrogate}; use super::calvin_fold_tests::{TARGET, TID, balance, calvin_redo, core_with_sum, sum_targets}; -use crate::bridge::envelope::{Response, Status}; +use super::test_commit::failing_install_sub_record; +use crate::bridge::envelope::{ErrorCode, Response, Status}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; use crate::data::executor::handlers::control::calvin::CalvinExecCtx; +use crate::data::executor::handlers::control::calvin::test_support::{ + bulk_delete_plan, point_insert_plan, seed_row, +}; +use crate::data::executor::handlers::control::calvin_reply::CalvinReply; use crate::data::executor::handlers::control::calvin_synthetic_txn_id; use crate::data::executor::task::ExecutionTask; +use crate::engine::document::store::{CollectionConfig, IndexPath}; use crate::types::{DatabaseId, Lsn, TenantId}; +use crate::wal::RedoRecord; const ORDERS: &str = "orders"; /// The vShard `make_default_task` homes to. @@ -209,3 +216,169 @@ fn a_calvin_install_keeps_fold_rows_in_the_write_set() { installed.write_set ); } + +/// The install applies the resolved record, not a replay of the staged +/// plans. A row that joins the bulk delete's predicate after the stage +/// stays, and the predicted row goes. +#[test] +fn a_calvin_install_applies_the_resolved_record_not_a_replay_of_the_plans() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + seed_row(&mut core, ORDERS, 1); + let resolved = stage_and_resolve(&mut core, &[bulk_delete_plan(ORDERS, Some(vec![1]))]); + seed_row(&mut core, ORDERS, 2); + + let plan = install_plan( + resolved.redo, + vec![ORDERS.to_string()], + Vec::new(), + 1, + resolved.reply, + ); + let installed = run_at(&mut core, &plan, 41); + + assert_eq!(installed.status, Status::Ok, "{:?}", installed.error_code); + assert!(base_row(&core, 1).is_none()); + assert!(base_row(&core, 2).is_some()); +} + +/// The install notes the index values of every document row at the +/// record's LSN. +#[test] +fn a_calvin_install_notes_index_values_at_the_redo_lsn() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let mut config = CollectionConfig::new(ORDERS); + config.index_paths.push(IndexPath::new("a")); + core.doc_configs.insert( + (DatabaseId::DEFAULT, TenantId::new(TID), ORDERS.to_string()), + config, + ); + let resolved = stage_and_resolve(&mut core, &[point_insert_plan(ORDERS, "o1", 7)]); + + let plan = install_plan( + resolved.redo, + vec![ORDERS.to_string()], + Vec::new(), + 1, + resolved.reply, + ); + let installed = run_at(&mut core, &plan, 43); + + assert_eq!(installed.status, Status::Ok, "{:?}", installed.error_code); + assert_eq!( + core.write_index.index_values.value_lsn( + DatabaseId::DEFAULT, + TenantId::new(TID), + ORDERS, + "a", + "1" + ), + Some(Lsn::new(43)), + "the install records the row's index value at the redo LSN" + ); +} + +/// An install whose record cannot decode answers an error and leaves +/// nothing of the record in base. +#[test] +fn a_calvin_install_whose_record_cannot_decode_answers_an_error() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let resolved = stage_and_resolve(&mut core, &[point_insert_plan(ORDERS, "o1", 7)]); + + let plan = install_plan( + vec![0xff, 0x00], + vec![ORDERS.to_string()], + Vec::new(), + 1, + resolved.reply, + ); + let installed = run_at(&mut core, &plan, 44); + + assert_eq!(installed.status, Status::Error); + assert!(!matches!( + installed.error_code.as_deref(), + Some(ErrorCode::OllpRetryRequired) + )); + assert!(base_row(&core, 7).is_none()); +} + +/// A refused install keeps the staged entry and its overlay, so a later +/// install of the record consumes them. +#[test] +fn a_refused_calvin_install_keeps_the_staged_entry() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let resolved = stage_and_resolve(&mut core, &[point_insert_plan(ORDERS, "o1", 7)]); + let mut failing = RedoRecord::from_bytes(&resolved.redo).expect("decode redo"); + failing.ops.push(failing_install_sub_record()); + let failing = failing.to_bytes().expect("encode redo"); + let synthetic = calvin_synthetic_txn_id(1, 0, VSHARD).expect("synthetic id"); + + let refused_plan = install_plan( + failing, + vec![ORDERS.to_string()], + Vec::new(), + 1, + resolved.reply.clone(), + ); + let refused = run_at(&mut core, &refused_plan, 45); + + assert!( + matches!( + refused.error_code.as_deref(), + Some(ErrorCode::RetryableRefusal { .. }) + ), + "{:?}", + refused.error_code + ); + assert!(core.calvin.commit_pending.contains_key(&(1, 0, VSHARD))); + assert!(core.txn_overlays.contains_key(&synthetic)); + assert!(base_row(&core, 7).is_none()); + + let plan = install_plan( + resolved.redo, + vec![ORDERS.to_string()], + Vec::new(), + 1, + resolved.reply, + ); + let installed = run_at(&mut core, &plan, 46); + + assert_eq!(installed.status, Status::Ok, "{:?}", installed.error_code); + assert!(base_row(&core, 7).is_some()); + assert!(!core.calvin.commit_pending.contains_key(&(1, 0, VSHARD))); + assert!(!core.txn_overlays.contains_key(&synthetic)); +} + +/// A reply that fails to render after a successful install leaves the +/// install `Ok` and carries the render error for the statement. +#[test] +fn a_reply_render_error_leaves_the_calvin_install_ok() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let resolved = stage_and_resolve(&mut core, &[point_insert_plan(ORDERS, "o1", 7)]); + let unrenderable = CalvinReplySpec::from(&CalvinReply::unrenderable_for_test(ORDERS)); + + let plan = install_plan( + resolved.redo, + vec![ORDERS.to_string()], + Vec::new(), + 1, + unrenderable, + ); + let installed = run_at(&mut core, &plan, 47); + + assert_eq!(installed.status, Status::Ok); + assert!( + matches!( + installed.error_code.as_deref(), + Some(ErrorCode::Internal { .. }) + ), + "{:?}", + installed.error_code + ); + assert!(base_row(&core, 7).is_some()); + assert!(!core.calvin.commit_pending.contains_key(&(1, 0, VSHARD))); +} diff --git a/nodedb/src/event/cdc/consume/remote.rs b/nodedb/src/event/cdc/consume/remote.rs index a65e5ca48..3cdcff989 100644 --- a/nodedb/src/event/cdc/consume/remote.rs +++ b/nodedb/src/event/cdc/consume/remote.rs @@ -19,7 +19,6 @@ use crate::control::server::shared::ddl::neutral::consumer_group::identity::cano use crate::control::state::SharedState; use crate::event::cdc::event::CdcEvent; use crate::event::cdc::offset::CdcOffset; -use crate::event::cdc::position::vshard_of_partition; use super::error::ConsumeError; use super::params::{ConsumeParams, ConsumeResult, batch_tails}; @@ -133,7 +132,7 @@ fn partition_members( .as_ref() .ok_or(ConsumeError::NoClusterRouting)?; let routing = routing_lock.read().unwrap_or_else(|p| p.into_inner()); - let Ok(group_id) = routing.group_for_vshard(vshard_of_partition(partition_id)) else { + let Ok(group_id) = routing.group_for_vshard(partition_id) else { return Ok(None); }; Ok(routing.group_info(group_id).map(|group| PartitionMembers { diff --git a/nodedb/src/event/cdc/consume/sink.rs b/nodedb/src/event/cdc/consume/sink.rs index bb5e9250f..b959c3263 100644 --- a/nodedb/src/event/cdc/consume/sink.rs +++ b/nodedb/src/event/cdc/consume/sink.rs @@ -18,7 +18,6 @@ use std::sync::Arc; use crate::control::state::SharedState; use crate::event::cdc::event::CdcEvent; -use crate::event::cdc::position::vshard_of_partition; use super::error::ConsumeError; use super::local::consume_stream; @@ -76,9 +75,9 @@ impl ReadPlan { } } - /// The data group of `partition`. + /// The data group of `partition`, a vShard. fn group_of(&self, partition: u32) -> Option { - let vshard = usize::try_from(vshard_of_partition(partition)).ok()?; + let vshard = usize::try_from(partition).ok()?; self.vshard_group.get(vshard).copied() } @@ -238,7 +237,6 @@ fn empty() -> ConsumeResult { #[cfg(test)] mod tests { use super::*; - use crate::event::cdc::position::calvin_partition; fn event(partition: u32) -> CdcEvent { CdcEvent { @@ -278,9 +276,8 @@ mod tests { assert!(plan.keeps(&plan.local, &event(0))); assert!(!plan.keeps(&plan.local, &event(1))); assert!(plan.keeps(&plan.remote[&2], &event(1))); - // A Calvin partition belongs to its vShard's group. - assert!(plan.keeps(&plan.remote[&3], &event(calvin_partition(2)))); - assert!(!plan.keeps(&plan.remote[&2], &event(calvin_partition(2)))); + assert!(plan.keeps(&plan.remote[&3], &event(2))); + assert!(!plan.keeps(&plan.remote[&2], &event(2))); } #[test] diff --git a/nodedb/src/event/cdc/offset.rs b/nodedb/src/event/cdc/offset.rs index 6595bbb56..32e10d562 100644 --- a/nodedb/src/event/cdc/offset.rs +++ b/nodedb/src/event/cdc/offset.rs @@ -11,13 +11,12 @@ //! metadata entry is the one event that raises it. No path proposes one, so //! every position has epoch `0`. //! - `index` names the write that produced the event. In a cluster it is the -//! Raft log index of the data-group entry, or the Calvin sequencer epoch of a -//! Calvin transaction. On a single node it is the WAL LSN of the write's -//! record. On a durable topic it is the message's log position. -//! - `sequence` orders the events of one write. Its low 32 bits carry -//! `2 × ordinal` for a data event and `2 × ordinal + 1` for that event's -//! late-data correction. Its high 32 bits carry the Calvin transaction's -//! position within its sequencer epoch, `0` for every other write. +//! Raft log index of the data-group entry. On a single node it is the WAL +//! LSN of the write's record. On a durable topic it is the message's log +//! position. +//! - `sequence` orders the events of one write. It carries `2 × ordinal` for +//! a data event and `2 × ordinal + 1` for that event's late-data +//! correction. //! //! Every replica applies a write at the same position, so a cursor from one //! node is valid on every node. @@ -29,6 +28,7 @@ use serde::{Deserialize, Serialize}; /// Bits of `sequence` that carry the ordinal and correction flag. const ORDINAL_BITS: u32 = 32; +/// The largest sequence a data event or its correction takes. const ORDINAL_MASK: u64 = (1 << ORDINAL_BITS) - 1; /// A lossless position in a CDC stream partition. @@ -104,25 +104,14 @@ impl CdcOffset { } /// The position of the data event with ordinal `ordinal` (1-based) in the - /// write at `(epoch, index)`, whose sequence carries `base` in its high - /// bits. - pub const fn data_event_in(epoch: u64, index: u64, base: u64, ordinal: u64) -> Self { - let low = if ordinal > (ORDINAL_MASK >> 1) { + /// write at `(epoch, index)`. + pub const fn data_event(epoch: u64, index: u64, ordinal: u64) -> Self { + let sequence = if ordinal > (ORDINAL_MASK >> 1) { ORDINAL_MASK - 1 } else { ordinal * 2 }; - Self::at(epoch, index, (base << ORDINAL_BITS) | low) - } - - /// [`Self::data_event_in`] with base `0`. - pub const fn data_event(epoch: u64, index: u64, ordinal: u64) -> Self { - Self::data_event_in(epoch, index, 0, ordinal) - } - - /// The high bits of the sequence: the Calvin position, `0` otherwise. - pub const fn base(self) -> u64 { - self.sequence >> ORDINAL_BITS + Self::at(epoch, index, sequence) } /// The ordinal of the data event this position belongs to. A correction @@ -211,16 +200,6 @@ mod tests { assert_eq!(second.ordinal(), 2); } - #[test] - fn calvin_positions_order_by_transaction_then_ordinal() { - let early = CdcOffset::data_event_in(0, 7, 3, 9); - let late = CdcOffset::data_event_in(0, 7, 4, 1); - assert!(early < late); - assert_eq!(late.base(), 4); - assert_eq!(late.ordinal(), 1); - assert!(early.correction() < late); - } - #[test] fn equal_positions_from_two_replicas_compare_equal() { let leader = CdcOffset::data_event(2, 42, 3); diff --git a/nodedb/src/event/cdc/position/ledger.rs b/nodedb/src/event/cdc/position/ledger.rs index a923aaea1..ac811f0e2 100644 --- a/nodedb/src/event/cdc/position/ledger.rs +++ b/nodedb/src/event/cdc/position/ledger.rs @@ -4,15 +4,14 @@ //! //! A Data-Plane change event names its write by the local WAL LSN of the //! write's record. That LSN differs on every replica. The Raft log position -//! of the entry the record applies, or the Calvin sequencer position of the -//! transaction it applies, is the same on every replica, so the CDC router -//! positions events by it. +//! of the entry the record applies is the same on every replica, so the CDC +//! router positions events by it. A committed Calvin slice installs from a +//! data-group entry, so its records take that entry's position too. //! //! The write path records `record LSN -> position` before it hands the write //! to a core, so the entry exists before any event of the write can reach //! the Event Plane. Boot rebuilds the ledger from the WAL: a Raft entry's -//! records link to its `ChangePosition` marker by proposal key, and a Calvin -//! transaction's redo record carries its sequencer position itself. A WAL +//! records link to its `ChangePosition` marker by proposal key. A WAL //! catch-up that rebuilds an event after a restart therefore positions it as //! the ring did. @@ -23,7 +22,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use nodedb_wal::WalRecord; use nodedb_wal::record::RecordType; -use super::marker::{CalvinPosition, ChangePositionMarker, ReplicatedPosition, WritePosition}; +use super::marker::{ChangePositionMarker, ReplicatedPosition}; /// Records the ledger keeps before it evicts the lowest LSN. An event whose /// record was evicted before the Event Plane routed it takes its partition's @@ -33,7 +32,7 @@ pub const CHANGE_POSITION_CAPACITY: usize = 1 << 18; /// Local record LSN to replicated position, bounded. #[derive(Debug)] pub struct ChangePositionLedger { - positions: RwLock>, + positions: RwLock>, /// Record LSN to the commit HLC (wall nanoseconds) of the write the /// record belongs to. Every replica holds the same value for a /// replicated write, so change events date by it. @@ -64,16 +63,6 @@ impl ChangePositionLedger { /// `position`. pub fn record(&self, record_lsn: u64, position: ReplicatedPosition) { self.mark_replicated(); - self.insert(record_lsn, WritePosition::Raft(position)); - } - - /// Record that the WAL record at `record_lsn` applies the Calvin - /// transaction at `position`. - pub fn record_calvin(&self, record_lsn: u64, position: CalvinPosition) { - self.insert(record_lsn, WritePosition::Calvin(position)); - } - - fn insert(&self, record_lsn: u64, position: WritePosition) { let mut positions = self.positions.write().unwrap_or_else(|p| p.into_inner()); positions.insert(record_lsn, position); while positions.len() > self.capacity { @@ -102,7 +91,7 @@ impl ChangePositionLedger { } /// The replicated position of the record at `record_lsn`, if known. - pub fn get(&self, record_lsn: u64) -> Option { + pub fn get(&self, record_lsn: u64) -> Option { self.positions .read() .unwrap_or_else(|p| p.into_inner()) @@ -125,9 +114,7 @@ impl ChangePositionLedger { /// order. Returns how many records it positioned. /// /// A record whose header key names a preceding `ChangePosition` marker - /// takes that marker's Raft position. Otherwise a `TransactionRedo` - /// record that a Calvin flush wrote takes its stamp's sequencer - /// position. + /// takes that marker's Raft position. pub fn recover(&self, records: &[WalRecord]) -> usize { let mut by_key: HashMap = HashMap::new(); let mut positioned = 0; @@ -156,13 +143,6 @@ impl ChangePositionLedger { { self.record(record.header.lsn, *position); positioned += 1; - continue; - } - if record_type == Some(RecordType::TransactionRedo) - && let Some(position) = calvin_position(record) - { - self.record_calvin(record.header.lsn, position); - positioned += 1; } } if !by_key.is_empty() { @@ -183,16 +163,6 @@ impl ChangePositionLedger { } } -/// The sequencer position a Calvin flush stamped on a redo record. -fn calvin_position(record: &WalRecord) -> Option { - let redo = crate::wal::RedoRecord::from_bytes(&record.payload).ok()?; - let stamp = redo.calvin_stamp?; - Some(CalvinPosition { - sequencer_epoch: stamp.epoch, - position: stamp.position, - }) -} - #[cfg(test)] mod tests { use super::*; @@ -205,8 +175,8 @@ mod tests { } } - fn raft(log_index: u64) -> Option { - Some(WritePosition::Raft(position(log_index))) + fn raft(log_index: u64) -> Option { + Some(position(log_index)) } #[test] @@ -217,12 +187,6 @@ mod tests { assert!(ledger.is_replicated()); assert_eq!(ledger.get(100), raft(7)); assert_eq!(ledger.get(101), None); - let calvin = CalvinPosition { - sequencer_epoch: 4, - position: 2, - }; - ledger.record_calvin(102, calvin); - assert_eq!(ledger.get(102), Some(WritePosition::Calvin(calvin))); } fn record(record_type: RecordType, lsn: u64, apply_key: u64, payload: Vec) -> WalRecord { diff --git a/nodedb/src/event/cdc/position/marker.rs b/nodedb/src/event/cdc/position/marker.rs index 8150fac4b..fc6563258 100644 --- a/nodedb/src/event/cdc/position/marker.rs +++ b/nodedb/src/event/cdc/position/marker.rs @@ -13,22 +13,6 @@ pub struct ReplicatedPosition { pub log_index: u64, } -/// Where a Calvin transaction sits in the sequencer log. Every replica's -/// scheduler applies the transaction at the same value. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct CalvinPosition { - pub sequencer_epoch: u64, - /// Zero-based position within the sequencer epoch's batch. - pub position: u32, -} - -/// The replicated position of one locally applied write. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum WritePosition { - Raft(ReplicatedPosition), - Calvin(CalvinPosition), -} - /// Byte length of an encoded [`ChangePositionMarker`]. pub const MARKER_LEN: usize = 32; diff --git a/nodedb/src/event/cdc/position/mod.rs b/nodedb/src/event/cdc/position/mod.rs index 2a3c9078f..950a1acd4 100644 --- a/nodedb/src/event/cdc/position/mod.rs +++ b/nodedb/src/event/cdc/position/mod.rs @@ -4,15 +4,10 @@ pub mod availability; pub mod epoch; pub mod ledger; pub mod marker; -pub mod partition; pub mod sequencer; pub use availability::AvailabilityFloors; pub use epoch::{entry_position, record_install_floor}; pub use ledger::{CHANGE_POSITION_CAPACITY, ChangePositionLedger}; -pub use marker::{ - CalvinPosition, ChangePositionMarker, MARKER_LEN, MarkerDecodeError, ReplicatedPosition, - WritePosition, -}; -pub use partition::{CALVIN_PARTITION_BASE, calvin_partition, vshard_of_partition}; +pub use marker::{ChangePositionMarker, MARKER_LEN, MarkerDecodeError, ReplicatedPosition}; pub use sequencer::{IndexSource, PartitionTail, PositionSequencer}; diff --git a/nodedb/src/event/cdc/position/partition.rs b/nodedb/src/event/cdc/position/partition.rs deleted file mode 100644 index aba4b1a6c..000000000 --- a/nodedb/src/event/cdc/position/partition.rs +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Change-feed partitions. -//! -//! A vShard's Raft-applied writes and its Calvin transactions come from two -//! separate replicated logs. Every replica applies each log in order, but the -//! two interleave differently on each replica, so they cannot share one -//! position order. Each log therefore has its own partition: partition `v` -//! carries vShard `v`'s Raft-applied writes, and partition -//! `CALVIN_PARTITION_BASE + v` carries its Calvin transactions. - -/// First partition of the Calvin partitions. -pub const CALVIN_PARTITION_BASE: u32 = 1 << 16; - -/// The partition of vShard `vshard`'s Calvin transactions. -pub const fn calvin_partition(vshard: u32) -> u32 { - CALVIN_PARTITION_BASE + vshard -} - -/// The vShard a partition belongs to. -pub const fn vshard_of_partition(partition: u32) -> u32 { - partition % CALVIN_PARTITION_BASE -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn a_calvin_partition_maps_back_to_its_vshard() { - assert_eq!(vshard_of_partition(calvin_partition(17)), 17); - assert_eq!(vshard_of_partition(17), 17); - assert_ne!(calvin_partition(17), 17); - } -} diff --git a/nodedb/src/event/cdc/position/sequencer.rs b/nodedb/src/event/cdc/position/sequencer.rs index 660ff943d..c69af4324 100644 --- a/nodedb/src/event/cdc/position/sequencer.rs +++ b/nodedb/src/event/cdc/position/sequencer.rs @@ -24,9 +24,8 @@ use crate::event::cdc::offset::CdcOffset; /// Where the position of an event comes from. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum IndexSource { - /// A replicated write: a Raft entry at `(epoch, index)`, or a Calvin - /// transaction at sequencer epoch `index` and batch position `base`. - Replicated { epoch: u64, index: u64, base: u64 }, + /// A replicated write: a Raft entry at `(epoch, index)`. + Replicated { epoch: u64, index: u64 }, /// The event's write has no known replicated position on a node that /// positions by it. The event joins the partition's current write. Unmapped, @@ -73,28 +72,25 @@ impl PositionSequencer { }) }); let last = tail.position; - let (epoch, index, base, continues) = match source { - IndexSource::Replicated { epoch, index, base } => ( + let (epoch, index, continues) = match source { + IndexSource::Replicated { epoch, index } => ( epoch, index, - base, - (epoch, index, base) == (last.epoch, last.index, last.base()) - && record_lsn == tail.record_lsn, + (epoch, index) == (last.epoch, last.index) && record_lsn == tail.record_lsn, ), IndexSource::Local(index) => ( 0, index, - 0, (0, index) == (last.epoch, last.index) && record_lsn == tail.record_lsn, ), - IndexSource::Unmapped => (last.epoch, last.index, last.base(), true), + IndexSource::Unmapped => (last.epoch, last.index, true), }; let ordinal = if continues { last.ordinal().saturating_add(1) } else { 1 }; - let position = CdcOffset::data_event_in(epoch, index, base, ordinal); + let position = CdcOffset::data_event(epoch, index, ordinal); *tail = PartitionTail { position, record_lsn, @@ -111,46 +107,23 @@ mod tests { None } + fn entry(index: u64) -> IndexSource { + IndexSource::Replicated { epoch: 0, index } + } + #[test] fn events_of_one_write_take_consecutive_ordinals() { let sequencer = PositionSequencer::new(); assert_eq!( - sequencer.next( - 0, - IndexSource::Replicated { - epoch: 0, - index: 7, - base: 0 - }, - 70, - none - ), + sequencer.next(0, entry(7), 70, none), CdcOffset::data_event(0, 7, 1) ); assert_eq!( - sequencer.next( - 0, - IndexSource::Replicated { - epoch: 0, - index: 7, - base: 0 - }, - 70, - none - ), + sequencer.next(0, entry(7), 70, none), CdcOffset::data_event(0, 7, 2) ); assert_eq!( - sequencer.next( - 0, - IndexSource::Replicated { - epoch: 0, - index: 8, - base: 0 - }, - 71, - none - ), + sequencer.next(0, entry(8), 71, none), CdcOffset::data_event(0, 8, 1) ); } @@ -158,27 +131,9 @@ mod tests { #[test] fn partitions_number_independently() { let sequencer = PositionSequencer::new(); - sequencer.next( - 0, - IndexSource::Replicated { - epoch: 0, - index: 7, - base: 0, - }, - 70, - none, - ); + sequencer.next(0, entry(7), 70, none); assert_eq!( - sequencer.next( - 1, - IndexSource::Replicated { - epoch: 0, - index: 7, - base: 0 - }, - 70, - none - ), + sequencer.next(1, entry(7), 70, none), CdcOffset::data_event(0, 7, 1) ); } @@ -186,16 +141,7 @@ mod tests { #[test] fn an_unmapped_event_joins_the_current_write() { let sequencer = PositionSequencer::new(); - sequencer.next( - 0, - IndexSource::Replicated { - epoch: 0, - index: 9, - base: 0, - }, - 90, - none, - ); + sequencer.next(0, entry(9), 90, none); assert_eq!( sequencer.next(0, IndexSource::Unmapped, 95, none), CdcOffset::data_event(0, 9, 2) @@ -214,16 +160,7 @@ mod tests { }) }; assert_eq!( - sequencer.next( - 0, - IndexSource::Replicated { - epoch: 0, - index: 12, - base: 0 - }, - 500, - seed - ), + sequencer.next(0, entry(12), 500, seed), CdcOffset::data_event(0, 12, 3) ); } @@ -239,29 +176,11 @@ mod tests { }; // Entry 12 applied again after a restart writes record 900. assert_eq!( - sequencer.next( - 0, - IndexSource::Replicated { - epoch: 0, - index: 12, - base: 0 - }, - 900, - seed - ), + sequencer.next(0, entry(12), 900, seed), CdcOffset::data_event(0, 12, 1) ); assert_eq!( - sequencer.next( - 0, - IndexSource::Replicated { - epoch: 0, - index: 12, - base: 0 - }, - 900, - none - ), + sequencer.next(0, entry(12), 900, none), CdcOffset::data_event(0, 12, 2) ); } @@ -275,26 +194,8 @@ mod tests { for _ in 0..events { // Each replica holds the entry at its own local WAL LSN. assert_eq!( - leader.next( - 2, - IndexSource::Replicated { - epoch: 0, - index, - base: 0 - }, - index * 10, - none - ), - follower.next( - 2, - IndexSource::Replicated { - epoch: 0, - index, - base: 0 - }, - index * 1_000, - none - ) + leader.next(2, entry(index), index * 10, none), + follower.next(2, entry(index), index * 1_000, none) ); } } @@ -303,19 +204,13 @@ mod tests { #[test] fn a_group_move_never_makes_positions_go_backwards() { let sequencer = PositionSequencer::new(); - let old_group = |index| IndexSource::Replicated { - epoch: 0, - index, - base: 0, - }; - let before = sequencer.next(4, old_group(9_000), 1, none); + let before = sequencer.next(4, entry(9_000), 1, none); // The new group numbers its log from a low index, in a higher epoch. let after = sequencer.next( 4, IndexSource::Replicated { epoch: 55, index: 3, - base: 0, }, 2, none, @@ -323,21 +218,4 @@ mod tests { assert!(after > before); assert_eq!(after, CdcOffset::data_event(55, 3, 1)); } - - #[test] - fn calvin_transactions_of_one_epoch_number_by_position() { - let sequencer = PositionSequencer::new(); - let calvin = |position| IndexSource::Replicated { - epoch: 0, - index: 12, - base: position, - }; - let first = sequencer.next(9, calvin(0), 100, none); - let second = sequencer.next(9, calvin(0), 100, none); - let next_txn = sequencer.next(9, calvin(1), 101, none); - assert_eq!(first, CdcOffset::data_event_in(0, 12, 0, 1)); - assert_eq!(second, CdcOffset::data_event_in(0, 12, 0, 2)); - assert_eq!(next_txn, CdcOffset::data_event_in(0, 12, 1, 1)); - assert!(first < second && second < next_txn); - } } diff --git a/nodedb/src/event/cdc/router.rs b/nodedb/src/event/cdc/router.rs index 685dff8c3..ce9833a53 100644 --- a/nodedb/src/event/cdc/router.rs +++ b/nodedb/src/event/cdc/router.rs @@ -23,7 +23,6 @@ use super::lag_warner::{CdcLagWarner, DEFAULT_THRESHOLD}; use super::offset::CdcOffset; use super::position::{ AvailabilityFloors, ChangePositionLedger, IndexSource, PartitionTail, PositionSequencer, - WritePosition, calvin_partition, }; use super::registry::StreamRegistry; use super::stream_def::{ChangeStreamDef, LateDataPolicy}; @@ -254,24 +253,15 @@ impl CdcRouter { /// The partition and position source of a write of `vshard` at local /// record `record_lsn`: the replicated position the record applies, when - /// the ledger knows it. A Calvin transaction goes to the vShard's Calvin - /// partition. The trigger action lane positions its events by it too. + /// the ledger knows it. The partition is the vShard. The trigger action + /// lane positions its events by it too. pub(crate) fn index_source(&self, vshard: u32, record_lsn: u64) -> (u32, IndexSource) { match self.positions.get(record_lsn) { - Some(WritePosition::Raft(position)) => ( + Some(position) => ( vshard, IndexSource::Replicated { epoch: position.epoch, index: position.log_index, - base: 0, - }, - ), - Some(WritePosition::Calvin(position)) => ( - calvin_partition(vshard), - IndexSource::Replicated { - epoch: 0, - index: position.sequencer_epoch, - base: u64::from(position.position), }, ), None if self.positions.is_replicated() => (vshard, IndexSource::Unmapped), @@ -715,45 +705,6 @@ mod tests { assert_eq!(positions(&leader, "s"), positions(&follower, "s")); } - #[test] - fn replicas_number_a_calvin_transaction_alike_in_its_own_partition() { - use crate::event::cdc::position::{CalvinPosition, calvin_partition}; - let replica = || { - let registry = Arc::new(StreamRegistry::new()); - registry.register(sample_def("s", "orders")); - CdcRouter::new(registry) - }; - let leader = replica(); - let follower = replica(); - let wt = test_tracker(); - let txn = CalvinPosition { - sequencer_epoch: 31, - position: 2, - }; - for (router, local_lsn) in [(&leader, 300), (&follower, 8_000)] { - router.positions().record_calvin(local_lsn, txn); - router.route_event(&write_at(local_lsn, 1), &wt); - router.route_event(&write_at(local_lsn, 2), &wt); - } - let seen = |router: &CdcRouter| { - router - .get_buffer(DatabaseId::new(7), 1, "s") - .expect("buffer") - .read_from(CdcOffset::ZERO, 16) - .iter() - .map(|event| (event.partition, event.position())) - .collect::>() - }; - assert_eq!( - seen(&leader), - vec![ - (calvin_partition(0), CdcOffset::data_event_in(0, 31, 2, 1)), - (calvin_partition(0), CdcOffset::data_event_in(0, 31, 2, 2)), - ] - ); - assert_eq!(seen(&leader), seen(&follower)); - } - #[test] fn a_node_without_raft_positions_by_local_wal_lsn() { let registry = Arc::new(StreamRegistry::new()); diff --git a/nodedb/src/event/cross_shard/types.rs b/nodedb/src/event/cross_shard/types.rs index 0efd27f9b..2d02699da 100644 --- a/nodedb/src/event/cross_shard/types.rs +++ b/nodedb/src/event/cross_shard/types.rs @@ -96,9 +96,9 @@ impl CrossShardWriteResponse { pub struct NotifyBroadcastMsg { /// The forwarding node. pub source_node: u64, - /// [`NOTIFY_PARTITION_GROUP`] or [`NOTIFY_PARTITION_CALVIN`]. + /// [`NOTIFY_PARTITION_GROUP`]. A receiver drops a run of any other kind. pub partition_kind: u8, - /// The data group, or the Calvin vShard. + /// The data group. pub partition_id: u64, pub after: crate::event::cdc::CdcOffset, pub through: crate::event::cdc::CdcOffset, @@ -107,8 +107,6 @@ pub struct NotifyBroadcastMsg { /// `NotifyBroadcastMsg::partition_kind` of a data group's feed. pub const NOTIFY_PARTITION_GROUP: u8 = 0; -/// `NotifyBroadcastMsg::partition_kind` of a vShard's Calvin feed. -pub const NOTIFY_PARTITION_CALVIN: u8 = 1; /// One forwarded change at its position. #[derive(Debug, Clone, zerompk::ToMessagePack, zerompk::FromMessagePack)] diff --git a/nodedb/src/event/topic/committed/deliver.rs b/nodedb/src/event/topic/committed/deliver.rs index 25f7eb0cf..e890dcc88 100644 --- a/nodedb/src/event/topic/committed/deliver.rs +++ b/nodedb/src/event/topic/committed/deliver.rs @@ -349,7 +349,6 @@ mod tests { partition: PARTITION, epoch: 0, index, - base: 0, }); message } diff --git a/nodedb/src/event/topic/committed/key.rs b/nodedb/src/event/topic/committed/key.rs index e82a2d956..6738322e0 100644 --- a/nodedb/src/event/topic/committed/key.rs +++ b/nodedb/src/event/topic/committed/key.rs @@ -6,13 +6,11 @@ //! A message is named by the change-feed partition of the record that carries //! it and a position in that partition: the record's replicated position with //! the message's ordinal. The apply that installs the record stamps that -//! position on the message itself ([`crate::wal::PublishPosition`]): a -//! Raft-applied record takes its entry's `(epoch, log index)`, and a Calvin -//! record takes its sequencer epoch and batch position. +//! position on the message itself ([`crate::wal::PublishPosition`]): the +//! record takes its data-group entry's `(epoch, log index)`. use crate::control::state::SharedState; use crate::event::cdc::CdcOffset; -use crate::event::cdc::position::vshard_of_partition; use crate::event::topic::types::PublishOrigin; use crate::wal::RedoPublish; @@ -53,12 +51,7 @@ pub(super) fn origin_of_event(ordinal: u32, publish: &RedoPublish) -> Option Option<(u64 let group_id = routing .read() .unwrap_or_else(|p| p.into_inner()) - .group_for_vshard(vshard_of_partition(partition)) + .group_for_vshard(partition) .ok()?; gate.leader_lease_term(group_id) .map(|term| (group_id, term)) @@ -108,7 +101,6 @@ mod tests { partition: 9, epoch: 2, index: 500, - base: 0, }); assert_eq!( origin_of_event(0, &stamped), diff --git a/nodedb/src/event/trigger/lane/fire.rs b/nodedb/src/event/trigger/lane/fire.rs index 57ff6ab33..454367518 100644 --- a/nodedb/src/event/trigger/lane/fire.rs +++ b/nodedb/src/event/trigger/lane/fire.rs @@ -288,7 +288,7 @@ async fn fire_partition( if delivery_lease(state, partition) != Some(lease) { break; } - let event = match action.to_event(partition, position) { + let event = match action.to_event(position) { Ok(event) => event, // A held event that does not rebuild fires nothing on any owner. Err(error) => { diff --git a/nodedb/src/event/trigger/lane/held.rs b/nodedb/src/event/trigger/lane/held.rs index b353141a3..96285a044 100644 --- a/nodedb/src/event/trigger/lane/held.rs +++ b/nodedb/src/event/trigger/lane/held.rs @@ -111,13 +111,13 @@ impl HeldAction { } /// The event the lane fires the actions of: this held event at - /// `position` of `partition`. + /// `position` of its partition. /// /// Its `lsn` and `sequence` carry the position's action identity, which /// every replica shares. The actions name their source write by them, so /// a body fired again by a later owner carries the same key and applies /// once. - pub fn to_event(&self, partition: u32, position: CdcOffset) -> crate::Result { + pub fn to_event(&self, position: CdcOffset) -> crate::Result { let source = EventSource::from_wal_code(self.source).ok_or_else(|| crate::Error::Internal { detail: format!( @@ -131,7 +131,7 @@ impl HeldAction { self.collection, self.op ), })?; - let (source_lsn, source_sequence) = action_identity(partition, position); + let (source_lsn, source_sequence) = action_identity(position); Ok(WriteEvent { sequence: source_sequence, collection: Arc::from(self.collection.as_str()), @@ -189,7 +189,7 @@ mod tests { let back = HeldAction::from_bytes(&held.to_bytes().expect("encode")).expect("decode"); assert_eq!(back, held); let fired = back - .to_event(7, CdcOffset::data_event(0, 12, 1)) + .to_event(CdcOffset::data_event(0, 12, 1)) .expect("rebuild"); assert_eq!(fired.op, WriteOp::Update); assert_eq!(fired.row_id.as_str(), "o-1"); diff --git a/nodedb/src/event/trigger/lane/position.rs b/nodedb/src/event/trigger/lane/position.rs index 77d308374..dc3c42756 100644 --- a/nodedb/src/event/trigger/lane/position.rs +++ b/nodedb/src/event/trigger/lane/position.rs @@ -4,8 +4,8 @@ //! //! The lane names an event as the CDC router does: by the change-feed //! partition of its write and a position in that partition. The position is -//! the write's replicated position (a Raft entry, or a Calvin transaction) -//! with the event's ordinal among the write's firing events. One partition's +//! the write's replicated position (a Raft entry) with the event's ordinal +//! among the write's firing events. One partition's //! events come from one Data-Plane core in apply order, so every replica //! numbers them alike. //! @@ -13,15 +13,10 @@ //! holds an action for its collection. The numbering then never depends on //! when a replica applied a trigger's DDL. -use crate::event::cdc::position::{CALVIN_PARTITION_BASE, PartitionTail, PositionSequencer}; +use crate::event::cdc::position::{PartitionTail, PositionSequencer}; use crate::event::cdc::{CdcOffset, CdcRouter}; use crate::event::types::WriteEvent; -/// The bit of an action identity's source LSN that marks a Calvin -/// partition. A Raft log index and a Calvin sequencer epoch share one number -/// space, so the bit keeps their events apart. Neither reaches `2^63`. -const CALVIN_IDENTITY_BIT: u64 = 1 << 63; - /// The position allocator of the lane's firing events. #[derive(Debug, Default)] pub struct ActionPositions { @@ -58,31 +53,21 @@ impl ActionPositions { } /// The `(source_lsn, source_sequence)` a fired action names its event by: -/// the position's index and sequence, with the Calvin bit set for a Calvin -/// partition. Every replica derives the same pair. +/// the position's index and sequence. Every replica derives the same pair. /// /// Every position has epoch `0` (see `cdc::offset`), so the pair names the /// event within its vShard. -pub fn action_identity(partition: u32, position: CdcOffset) -> (u64, u64) { - let kind = if partition >= CALVIN_PARTITION_BASE { - CALVIN_IDENTITY_BIT - } else { - 0 - }; - (position.index | kind, position.sequence) +pub fn action_identity(position: CdcOffset) -> (u64, u64) { + (position.index, position.sequence) } #[cfg(test)] mod tests { use super::*; - use crate::event::cdc::position::calvin_partition; #[test] - fn a_raft_and_a_calvin_event_at_one_index_have_distinct_identities() { + fn an_action_identity_is_the_index_and_sequence() { let position = CdcOffset::data_event(0, 12, 1); - let raft = action_identity(3, position); - let calvin = action_identity(calvin_partition(3), position); - assert_eq!(raft, (12, position.sequence)); - assert_ne!(raft, calvin); + assert_eq!(action_identity(position), (12, position.sequence)); } } diff --git a/nodedb/src/event/trigger/lane/snapshot.rs b/nodedb/src/event/trigger/lane/snapshot.rs index ae1171a86..d2161cda1 100644 --- a/nodedb/src/event/trigger/lane/snapshot.rs +++ b/nodedb/src/event/trigger/lane/snapshot.rs @@ -18,7 +18,6 @@ use std::time::Duration; use crate::control::state::SharedState; use crate::event::cdc::CdcOffset; -use crate::event::cdc::position::calvin_partition; use crate::event::topic::committed::cursor::{delivered_through, raise_delivered_here}; use crate::event::topic::types::PublishOrigin; use crate::types::snapshot::{GroupEventLane, HeldAtPosition, PartitionCursor}; @@ -35,13 +34,9 @@ fn lane_error(detail: impl std::fmt::Display) -> crate::Error { } } -/// Every partition of `vshards`: each vShard's Raft partition and its Calvin -/// partition, in order. +/// Every partition of `vshards`, in order: each vShard is one partition. fn partitions_of(vshards: &HashSet) -> Vec { - let mut partitions: Vec = vshards - .iter() - .flat_map(|vshard| [*vshard, calvin_partition(*vshard)]) - .collect(); + let mut partitions: Vec = vshards.iter().copied().collect(); partitions.sort_unstable(); partitions } @@ -104,10 +99,8 @@ pub async fn capture(state: &SharedState, vshards: &HashSet) -> crate::Resu /// Install `lane`, a snapshot of `vshards` cut at `cut_index`, on this node. /// -/// A Raft partition's rows at or below the cut are the builder's: this node -/// never applies those entries. A Calvin partition's rows join this node's, -/// since its Calvin applies continue apart from the data group. Each cursor -/// rises to the builder's. +/// A partition's rows at or below the cut are the builder's: this node never +/// applies those entries. Each cursor rises to the builder's. pub fn install( state: &SharedState, vshards: &HashSet, @@ -131,13 +124,11 @@ pub fn install( (CdcOffset::at(*epoch, *index, *sequence), bytes.clone()) }) .collect(); - let through = vshards - .contains(&partition) - .then_some(CdcOffset::whole_write(0, cut_index)); - ledgers - .actions - .ledger - .replace_through(partition, through, &rows)?; + ledgers.actions.ledger.replace_through( + partition, + Some(CdcOffset::whole_write(0, cut_index)), + &rows, + )?; } for (partition, epoch, index, sequence, bytes) in held_publishes { let publish: RedoPublish = zerompk::from_msgpack(&bytes).map_err(lane_error)?; @@ -162,12 +153,9 @@ mod tests { use super::*; #[test] - fn a_group_covers_the_raft_and_calvin_partitions_of_its_vshards() { + fn a_group_covers_the_partitions_of_its_vshards_in_order() { let vshards: HashSet = [3, 1].into_iter().collect(); - assert_eq!( - partitions_of(&vshards), - vec![1, 3, calvin_partition(1), calvin_partition(3)] - ); + assert_eq!(partitions_of(&vshards), vec![1, 3]); } #[test] diff --git a/nodedb/src/event/wal_replay.rs b/nodedb/src/event/wal_replay.rs index 388d1369f..a3cceeb54 100644 --- a/nodedb/src/event/wal_replay.rs +++ b/nodedb/src/event/wal_replay.rs @@ -726,16 +726,6 @@ mod tests { assert_eq!(seq, 1, "the VectorPut sub-op did not consume a sequence"); } - /// A `CalvinApplied` payload-free marker emits no events — its base writes, - /// if any, ride a separate `TransactionRedo`. - #[test] - fn calvin_applied_marker_emits_no_events() { - let record = make_record(RecordType::CalvinApplied, &[], 1, 0, 302); - let mut seq = 0u64; - assert!(record_to_events(&record, &mut seq).is_empty()); - assert_eq!(seq, 0); - } - /// A malformed `TransactionRedo` payload is skipped (logged, no panic) and /// produces no events. #[test] diff --git a/nodedb/src/event/wal_replay_kind.rs b/nodedb/src/event/wal_replay_kind.rs index 0ad4f4b43..c2486408f 100644 --- a/nodedb/src/event/wal_replay_kind.rs +++ b/nodedb/src/event/wal_replay_kind.rs @@ -48,12 +48,6 @@ pub(super) fn row_kind(record_type: RecordType) -> Option { // the same events. A columnar insert rides the same record type and // rebuilds none: its forward path emits no WriteEvent. RecordType::TimeseriesBatch => Some(RowRecord::Timeseries), - // `CalvinApplied` is a payload-free applied-marker: it records that a - // sequencer `(epoch, position)` was applied, but carries no writes. Its - // base writes, if any, ride a separate `TransactionRedo`; a pure-read or - // CRDT-only commit has no base WriteEvents at all (CRDT effects ride - // `CrdtDelta` records). Nothing to emit. - RecordType::CalvinApplied => None, // The records below carry NO forward-path Data-Plane WriteEvent, so // there is nothing for replay to reconstruct. `record_to_events` // reconstructs exactly the forward WriteEvent stream the Data Plane diff --git a/nodedb/src/main_boot/shutdown_wiring.rs b/nodedb/src/main_boot/shutdown_wiring.rs index c6a713093..c9f2b6e0b 100644 --- a/nodedb/src/main_boot/shutdown_wiring.rs +++ b/nodedb/src/main_boot/shutdown_wiring.rs @@ -62,7 +62,7 @@ pub(crate) fn wire_shutdown_bus( // Final WAL fsync. Every append before this phase is buffered until an // fsync covers it, and a write whose caller never awaited durability - // (a Calvin applied marker, a background maintenance record) has only + // (a background maintenance record) has only // this fsync between it and a graceful exit. Registered at startup so the // bus cannot pass `WalFsync` without it. spawn_wal_fsync_barrier(Arc::clone(shared), &shutdown_bus); diff --git a/nodedb/src/wal/manager/append_metadata.rs b/nodedb/src/wal/manager/append_metadata.rs index 7648f4e35..cf2799ea7 100644 --- a/nodedb/src/wal/manager/append_metadata.rs +++ b/nodedb/src/wal/manager/append_metadata.rs @@ -1,8 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 //! WAL appends for node-global metadata: temporal-purge audit, surrogate -//! allocation/binding, Calvin epoch tracking, applied Raft proposals, and -//! sync watermarks. +//! allocation/binding, applied Raft proposals, and sync watermarks. use nodedb_wal::record::RecordType; @@ -86,31 +85,6 @@ impl WalAppender<'_> { ) } - /// Append a `CalvinApplied` record after a Calvin executor successfully - /// commits a `MetaOp::CalvinExecute` batch. - /// - /// The scheduler's restart path scans these records to compute - /// `last_applied_epoch` for a given vshard without re-reading the full - /// Raft sequencer log. `vshard_id` is stored in the payload (not in the - /// WAL record header's `vshard_id` field) so it can be decoded during - /// a scan of all records regardless of which vshard they were written on. - pub fn append_calvin_applied( - &self, - vshard_id: crate::types::VShardId, - epoch: u64, - position: u32, - ) -> crate::Result { - let payload = - nodedb_wal::CalvinAppliedPayload::new(epoch, position, vshard_id.as_u32()).to_bytes(); - self.append_record( - RecordType::CalvinApplied, - TenantId::new(0), - vshard_id, - DatabaseId::DEFAULT, - &payload, - ) - } - /// Append a payload-free `ProposalApplied` marker carrying this /// appender's apply key. An apply that writes no record of its own /// appends it in place of its forward record, so the proposal's key still diff --git a/nodedb/src/wal/redo/record.rs b/nodedb/src/wal/redo/record.rs index 85b10cb9c..ddebbd4c6 100644 --- a/nodedb/src/wal/redo/record.rs +++ b/nodedb/src/wal/redo/record.rs @@ -109,10 +109,8 @@ pub struct RedoPublish { } /// The replicated position of the record that carries a committed message: -/// its change-feed partition, and `(epoch, index, base)` as the partition -/// orders writes. A Raft-applied record takes its entry's `(epoch, log -/// index)` with base `0`. A Calvin record takes its sequencer epoch as the -/// index and its batch position as the base. +/// its change-feed partition (the vShard), and its data-group entry's +/// `(epoch, log index)`. #[derive( Debug, Clone, @@ -129,7 +127,6 @@ pub struct PublishPosition { pub partition: u32, pub epoch: u64, pub index: u64, - pub base: u64, } /// Identity of one cross-shard trigger request: the source write's position @@ -177,8 +174,7 @@ pub struct RedoSubRecord { /// Calvin sequencer coordinates a [`RedoRecord`] carries when it installs a /// committed Calvin slice. The record doubles as the position's applied -/// marker: boot recovery reads the stamp. Mirrors -/// `nodedb_wal::CalvinAppliedPayload` in its coordinates. +/// marker: boot recovery reads the stamp. #[derive( Debug, Clone, @@ -427,7 +423,6 @@ mod tests { partition: 3, epoch: 0, index: 41, - base: 0, }), }], row_changes: Vec::new(), diff --git a/nodedb/src/wal/replay/surrogate/dispatch.rs b/nodedb/src/wal/replay/surrogate/dispatch.rs index 8f32809f6..524302d13 100644 --- a/nodedb/src/wal/replay/surrogate/dispatch.rs +++ b/nodedb/src/wal/replay/surrogate/dispatch.rs @@ -102,7 +102,6 @@ pub fn replay_surrogate_records( | RecordType::CollectionTombstoned | RecordType::TimeAnchor | RecordType::TemporalPurge - | RecordType::CalvinApplied // SyncSeqAdvance: not relevant to surrogate replay; the sync // idempotency replay pass handles HWM reconstruction. | RecordType::SyncSeqAdvance diff --git a/nodedb/src/wal/replay/sync_hwm/advance.rs b/nodedb/src/wal/replay/sync_hwm/advance.rs index 4352b3ca9..07087e37b 100644 --- a/nodedb/src/wal/replay/sync_hwm/advance.rs +++ b/nodedb/src/wal/replay/sync_hwm/advance.rs @@ -117,7 +117,6 @@ pub fn replay_sync_hwm_records( | RecordType::CollectionTombstoned | RecordType::TimeAnchor | RecordType::TemporalPurge - | RecordType::CalvinApplied | RecordType::SurrogateAlloc | RecordType::SurrogateBind | RecordType::FtsIndex diff --git a/nodedb/tests/inproc/cases/calvin_scheduler_restart.rs b/nodedb/tests/inproc/cases/calvin_scheduler_restart.rs index e745f51c4..af28640ed 100644 --- a/nodedb/tests/inproc/cases/calvin_scheduler_restart.rs +++ b/nodedb/tests/inproc/cases/calvin_scheduler_restart.rs @@ -3,10 +3,11 @@ //! Calvin scheduler restart idempotency tests. //! //! Covers the WAL-recovery layer: after N epochs, a freshly-opened -//! `WalManager` reads back the correct applied `(epoch, position)` markers, -//! including out-of-order writes, vshard isolation, the greenfield sentinel, -//! and — critically — a MULTI-POSITION epoch where only some positions -//! committed before the crash (a torn transaction must be re-applied, not lost). +//! `WalManager` reads back the correct applied `(epoch, position)` pairs from +//! the stamped `TransactionRedo` records, including out-of-order writes, +//! vshard isolation, the greenfield sentinel, and — critically — a +//! MULTI-POSITION epoch where only some positions committed before the crash +//! (a torn transaction must be re-applied, not lost). //! //! End-to-end scheduler rebuild via `MultiRaft::read_committed_entries` //! is covered by `nodedb-cluster/tests/calvin_3node_shard_failover.rs:: @@ -14,9 +15,14 @@ use tempfile::TempDir; -use nodedb::control::cluster::calvin::scheduler::{NOT_YET_APPLIED_EPOCH, read_applied_recovery}; -use nodedb::types::VShardId; +use nodedb::control::cluster::calvin::scheduler::{ + AppliedRecovery, NOT_YET_APPLIED_EPOCH, recover_all_applied, +}; +use nodedb::control::security::catalog::SystemCatalog; +use nodedb::event::EventSource; +use nodedb::types::{DatabaseId, TenantId, VShardId}; use nodedb::wal::manager::{NO_APPLY_KEY, WalManager}; +use nodedb::wal::{CalvinStamp, RedoRecord}; // ── Helper ──────────────────────────────────────────────────────────────────── @@ -24,11 +30,53 @@ fn open_wal(dir: &TempDir) -> WalManager { WalManager::open_for_testing(dir.path()).expect("open wal") } +/// Append the stamped redo record of the Calvin slice at `(epoch, position)` +/// on `vshard_id`: the WAL record a committed slice's install writes. +fn append_stamped(wal: &WalManager, vshard_id: u32, epoch: u64, position: u32) { + let record = RedoRecord { + version: 1, + ops: Vec::new(), + calvin_stamp: Some(CalvinStamp { + epoch, + position, + vshard_id, + }), + cross_shard_applied: None, + row_sources: Vec::new(), + publishes: Vec::new(), + row_changes: Vec::new(), + }; + wal.appender(NO_APPLY_KEY) + .with_event_source(EventSource::User) + .append_transaction_redo( + TenantId::new(0), + VShardId::new(vshard_id), + DatabaseId::DEFAULT, + &record, + ) + .expect("append stamped redo"); +} + +/// The recovered state of `vshard_id` from a catalog with nothing saved. A +/// vShard with no stamped redo record recovers as the greenfield state: the +/// `NOT_YET_APPLIED_EPOCH` sentinel and an empty tail. +fn recover_vshard(wal: &WalManager, vshard_id: u32) -> AppliedRecovery { + let catalog = SystemCatalog::open_in_memory().expect("in-memory catalog"); + recover_all_applied(wal, &catalog, &|_| None) + .expect("recovery scan should succeed") + .remove(&vshard_id) + .unwrap_or(AppliedRecovery { + fully_applied_epoch: NOT_YET_APPLIED_EPOCH, + applied_tail: Default::default(), + max_applied_epoch: NOT_YET_APPLIED_EPOCH, + }) +} + // ── Tests ───────────────────────────────────────────────────────────────────── -/// Simulating 5 epochs (each with a single position 0): append CalvinApplied +/// Simulating 5 epochs (each with a single position 0): append stamped redo /// records, sync, then verify that a freshly-opened WalManager reads back all -/// five markers and a max-applied epoch of 5. +/// five positions and a max-applied epoch of 5. #[test] fn scheduler_restart_reads_applied_markers_after_five_epochs() { let dir = TempDir::new().unwrap(); @@ -37,16 +85,14 @@ fn scheduler_restart_reads_applied_markers_after_five_epochs() { { let wal = open_wal(&dir); for epoch in 1u64..=5 { - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard_id), epoch, 0) - .unwrap(); + append_stamped(&wal, vshard_id, epoch, 0); } wal.sync().unwrap(); } { let wal = open_wal(&dir); - let rec = read_applied_recovery(&wal, vshard_id).expect("recovery scan should succeed"); + let rec = recover_vshard(&wal, vshard_id); assert_eq!(rec.max_applied_epoch, 5, "max applied epoch should be 5"); for epoch in 1u64..=5 { assert!( @@ -66,24 +112,15 @@ fn scheduler_restart_reports_max_and_all_positions_regardless_of_order() { { let wal = open_wal(&dir); - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard_id), 3, 0) - .unwrap(); - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard_id), 1, 0) - .unwrap(); - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard_id), 5, 0) - .unwrap(); - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard_id), 2, 0) - .unwrap(); + for epoch in [3u64, 1, 5, 2] { + append_stamped(&wal, vshard_id, epoch, 0); + } wal.sync().unwrap(); } { let wal = open_wal(&dir); - let rec = read_applied_recovery(&wal, vshard_id).unwrap(); + let rec = recover_vshard(&wal, vshard_id); assert_eq!( rec.max_applied_epoch, 5, "max should be 5, not last-written 2" @@ -112,19 +149,15 @@ fn scheduler_restart_multi_position_epoch_does_not_lose_uncommitted_position() { { let wal = open_wal(&dir); // Prior epoch fully applied. - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard_id), 8, 0) - .unwrap(); + append_stamped(&wal, vshard_id, 8, 0); // Torn epoch: position 0 committed, position 1 did NOT (crash between). - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(vshard_id), torn_epoch, 0) - .unwrap(); + append_stamped(&wal, vshard_id, torn_epoch, 0); wal.sync().unwrap(); } { let wal = open_wal(&dir); - let rec = read_applied_recovery(&wal, vshard_id).unwrap(); + let rec = recover_vshard(&wal, vshard_id); assert!( rec.applied_tail.contains(&(torn_epoch, 0)), "committed position 0 must stay applied (not re-applied on restart)" @@ -134,8 +167,7 @@ fn scheduler_restart_multi_position_epoch_does_not_lose_uncommitted_position() { "uncommitted position 1 must be reported NOT applied so it is \ re-delivered and applied on restart — else a lost/torn transaction" ); - // The fully-applied watermark stays below the torn epoch: the per-epoch - // collapse that caused the bug is gone. + // The fully-applied watermark stays below the torn epoch. assert!( rec.fully_applied_epoch == NOT_YET_APPLIED_EPOCH || rec.fully_applied_epoch < torn_epoch, @@ -144,7 +176,7 @@ fn scheduler_restart_multi_position_epoch_does_not_lose_uncommitted_position() { } } -/// Greenfield: a WAL with no CalvinApplied records returns the +/// Greenfield: a WAL with no stamped redo record returns the /// `NOT_YET_APPLIED_EPOCH` sentinel and an empty tail. Epoch 0 is a valid real /// epoch, so a distinct sentinel distinguishes "never applied" from "applied /// epoch 0". @@ -152,35 +184,29 @@ fn scheduler_restart_multi_position_epoch_does_not_lose_uncommitted_position() { fn scheduler_restart_greenfield_returns_sentinel() { let dir = TempDir::new().unwrap(); let wal = open_wal(&dir); - let rec = read_applied_recovery(&wal, 1).unwrap(); + let rec = recover_vshard(&wal, 1); assert_eq!(rec.fully_applied_epoch, NOT_YET_APPLIED_EPOCH); assert_eq!(rec.max_applied_epoch, NOT_YET_APPLIED_EPOCH); assert!(rec.applied_tail.is_empty()); } -/// Multiple vshards: recovery for one vshard must not see another's markers. +/// Multiple vshards: recovery for one vshard must not see another's records. #[test] fn scheduler_restart_vshard_isolation() { let dir = TempDir::new().unwrap(); { let wal = open_wal(&dir); - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(1), 10, 0) - .unwrap(); - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(2), 99, 0) - .unwrap(); - wal.appender(NO_APPLY_KEY) - .append_calvin_applied(VShardId::new(1), 20, 0) - .unwrap(); + append_stamped(&wal, 1, 10, 0); + append_stamped(&wal, 2, 99, 0); + append_stamped(&wal, 1, 20, 0); wal.sync().unwrap(); } { let wal = open_wal(&dir); - let r1 = read_applied_recovery(&wal, 1).unwrap(); - let r2 = read_applied_recovery(&wal, 2).unwrap(); + let r1 = recover_vshard(&wal, 1); + let r2 = recover_vshard(&wal, 2); assert_eq!(r1.max_applied_epoch, 20, "vshard 1 max is 20"); assert!(r1.applied_tail.contains(&(10, 0)) && r1.applied_tail.contains(&(20, 0))); assert!(!r1.applied_tail.contains(&(99, 0)), "must not see vshard 2"); From 50c78dc99d531325c4e242a282deed8b89e64b5a Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sat, 10 Oct 2026 08:59:09 +0800 Subject: [PATCH 07/19] fix(calvin): make stamped redo installs apply once across crashes and copies Redo sum targets now carry only the targets a source row folds on its own core; balances that travel on an ApplyBalanceDelta task no longer appear in the table, so the install skips them without a deferred list. Count redo copies that install nothing because their position was applied or another copy held the claim. Add a fail gate between a durable install and its RedoApplied event so tests can exercise a withheld event and a repeated proposal. Statements in a transaction block that write a materialized-sum source answer with the command tag of their own rows only. The derived balance move no longer changes the verb or the count. Add cluster, crash, wire, and unit tests for duplicate redo copies, replica convergence, and installs cut by a crash. --- .../cases/calvin_duplicate_redo_copy.rs | 198 ++++++++++++ .../cases/calvin_replica_content.rs | 283 ++++++++++++++++++ .../cases/calvin_replica_convergence.rs | 247 +++++++++++++++ .../tests/common_suite/cases/mod.rs | 3 + .../src/physical_plan/document/sum_target.rs | 10 +- .../node/inspect/core_placement.rs | 21 +- .../apply_loop/calvin_redo.rs | 61 ++++ .../apply_loop/transaction_redo.rs | 4 +- nodedb/src/control/fail_gate.rs | 86 +++--- .../server/native/dispatch/sql_fold.rs | 22 +- .../server/native/dispatch/sql_loop.rs | 44 +-- .../handler/routing/dispatch_loop/run.rs | 30 +- .../handler/routing/dispatch_loop/task.rs | 14 +- .../handler/routing/execute_dml_hooks.rs | 13 +- .../handler/routing/gateway_dispatch.rs | 18 +- .../pgwire/handler/routing/gateway_fold.rs | 25 +- .../server/response_shape/calvin_fold.rs | 17 +- .../response_shape/types/dml_outcome.rs | 234 +++++++++++++-- .../server/response_shape/types/mod.rs | 2 +- nodedb/src/control/state/calvin_counters.rs | 4 + nodedb/src/control/state/calvin_local.rs | 1 + .../decode/transaction_redo.rs | 1 - .../transaction_redo/sum_targets.rs | 83 ++++- .../redo_apply/calvin_fold_tests.rs | 45 +-- .../redo_apply/cross_shard_fold_tests.rs | 160 ++++++++++ .../transaction/redo_apply/document.rs | 14 +- .../handlers/transaction/redo_apply/mod.rs | 6 +- .../handlers/transaction/redo_apply/state.rs | 13 +- .../transaction/redo_apply/validate.rs | 2 +- .../data/executor/write_set_crash_tests.rs | 104 ++++++- nodedb/tests/crash_calvin_redo_cut.rs | 246 +++++++++++++++ .../cases/materialized_sum_txn_command_tag.rs | 127 ++++++++ nodedb/tests/wire/cases/mod.rs | 1 + 33 files changed, 1932 insertions(+), 207 deletions(-) create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_duplicate_redo_copy.rs create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_content.rs create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_convergence.rs create mode 100644 nodedb/src/data/executor/handlers/transaction/redo_apply/cross_shard_fold_tests.rs create mode 100644 nodedb/tests/crash_calvin_redo_cut.rs create mode 100644 nodedb/tests/wire/cases/materialized_sum_txn_command_tag.rs diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_duplicate_redo_copy.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_duplicate_redo_copy.rs new file mode 100644 index 000000000..50e53e18f --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_duplicate_redo_copy.rs @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A Calvin slice whose redo installed, while its scheduler has not heard +//! of the install, gets its redo proposed again. The second copy also +//! commits, and installs nothing: every replica applies each slice once. +//! +//! 1. A 3-node cluster holds `accts`, whose `balance` is the materialized +//! sum of `entries.amount`, and `entries`, on distinct vShards. `acc-0` +//! starts at 100. +//! 2. The fail gates `calvin::before_redo_applied_push::` and +//! `calvin::before_redo_applied_push::` withhold the `RedoApplied` +//! event of each slice's install. The apply concludes, so each data group +//! applies past the redo with no install heard, and the leader's stall +//! tick proposes the redo again. +//! 3. Once every node skipped a second copy, the test releases the gates. +//! 4. The client's COMMIT succeeds once. `acc-0` reads 140, not the 180 of a +//! second install. Every node installed each slice it replicates once, +//! and every replica holds the same content. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::sync::atomic::Ordering; +use std::time::Duration; + +use nodedb_types::fail_point::{FailAction, FailGuard}; + +use super::calvin_multishard_fixture::{Fixture, tags}; +use super::calvin_replica_content::{ + first_column, names_on_distinct_vshards, run_retrying, sum_ddl, wait_replicas_agree, +}; +use crate::common::cluster_harness::{TestClusterNode, wait_for}; + +/// How long the test waits for the first install, and for every node to +/// skip a second copy. +const HOLD_DEADLINE: Duration = Duration::from_secs(30); + +/// How long the replicas get to report their installs after the release. +const INSTALL_DEADLINE: Duration = Duration::from_secs(15); + +/// Committed Calvin slices whose stamped redo installed on `node`. +fn installs(node: &TestClusterNode) -> u64 { + node.shared + .calvin + .counters + .write_versions_recorded + .load(Ordering::Relaxed) +} + +/// Stamped redo copies `node` applied that installed nothing. +fn skipped_copies(node: &TestClusterNode) -> u64 { + node.shared + .calvin + .counters + .redo_copies_skipped + .load(Ordering::Relaxed) +} + +/// How many of `collections` `node` replicates: each is one slice of the +/// transaction. +fn slices_on(node: &TestClusterNode, collections: &[&str]) -> u64 { + let hosted = collections + .iter() + .filter(|collection| { + node.group_id_for_collection(collection) + .is_some_and(|group| node.replicates_data_group(group)) + }) + .count(); + u64::try_from(hosted).unwrap_or(u64::MAX) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_second_redo_copy_that_commits_installs_nothing() { + let [accts, entries] = names_on_distinct_vshards(["dup_redo_accts", "dup_redo_entries"]); + let collections = [accts.as_str(), entries.as_str()]; + let [accts_ddl, entries_ddl, sum] = sum_ddl(&entries, &accts); + let fx = Fixture::spawn(&[accts_ddl, entries_ddl]).await; + fx.cluster + .exec_ddl_on_any_leader(&sum) + .await + .unwrap_or_else(|e| panic!("{sum}: {e}")); + for collection in collections { + fx.wait_group_mounted(collection).await; + } + run_retrying( + &fx.cluster.nodes[0].client, + "seed the account", + &format!("INSERT INTO {accts} (id, owner, balance) VALUES ('acc-0', 'seed', '100')"), + ) + .await; + fx.converge().await; + + let installs_before: Vec = fx.cluster.nodes.iter().map(installs).collect(); + let skipped_before: Vec = fx.cluster.nodes.iter().map(skipped_copies).collect(); + let mut session = fx.raw().await; + let txn = format!( + "BEGIN; \ + INSERT INTO {entries} (id, account_id, amount) VALUES ('e1', 'acc-0', '25'); \ + INSERT INTO {entries} (id, account_id, amount) VALUES ('e2', 'acc-0', '15'); \ + COMMIT" + ); + + let gate_dir = tempfile::tempdir().expect("gate directory"); + let release = gate_dir.path().join("release"); + let parked = gate_dir.path().join("release.parked"); + let committed = { + let _gates = collections.map(|collection| { + FailGuard::install( + &format!("calvin::before_redo_applied_push::{collection}"), + FailAction::WaitForFile(release.clone()), + ) + }); + let commit = tags(&mut session, &txn); + let copy_again = async { + wait_for( + "a replica installs a slice and withholds its RedoApplied", + HOLD_DEADLINE, + Duration::from_millis(20), + || parked.exists(), + ) + .await; + wait_for( + "every node applies a second redo copy that installs nothing", + HOLD_DEADLINE, + Duration::from_millis(50), + || { + fx.cluster + .nodes + .iter() + .zip(&skipped_before) + .all(|(node, before)| skipped_copies(node) > *before) + }, + ) + .await; + std::fs::write(&release, b"release").expect("release the withheld events"); + }; + let (committed, ()) = tokio::join!(commit, copy_again); + committed + }; + + assert_eq!( + committed + .iter() + .filter(|tag| tag.as_str() == "COMMIT") + .count(), + 1, + "the client's COMMIT answers once, found tags {committed:?}" + ); + assert_eq!( + committed.last().map(String::as_str), + Some("COMMIT"), + "the transaction ends committed, found tags {committed:?}" + ); + + fx.converge().await; + wait_for( + "every node reports the install of each slice it replicates", + INSTALL_DEADLINE, + Duration::from_millis(50), + || { + fx.cluster + .nodes + .iter() + .zip(&installs_before) + .all(|(node, before)| installs(node) - before >= slices_on(node, &collections)) + }, + ) + .await; + for (node, before) in fx.cluster.nodes.iter().zip(&installs_before) { + assert_eq!( + installs(node) - before, + slices_on(node, &collections), + "node {} installs each slice it replicates once, whatever copies its log holds", + node.node_id + ); + } + + wait_replicas_agree(&fx.cluster, &collections).await; + let balance = first_column( + &fx.coordinator().client, + &format!("SELECT balance FROM {accts} WHERE id = 'acc-0'"), + ) + .await; + assert_eq!( + balance, + vec!["140".to_owned()], + "100 + 25 + 15 lands once; a second install of the balance slice reads 180" + ); + let rows = first_column( + &fx.coordinator().client, + &format!("SELECT id FROM {entries}"), + ) + .await; + assert_eq!(rows.len(), 2, "the entries slice wrote its two rows once"); + + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_content.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_content.rs new file mode 100644 index 000000000..c6eb12ead --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_content.rs @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Per-replica stored content for the Calvin redo convergence cases. +//! +//! A replica's content comes from the tenant snapshot of each Data-Plane +//! core of its node. The read never leaves the node, so it shows the rows +//! that replica installed, never a row a gateway fetched from a leader. +//! +//! A collection homes to one vShard. The content of a vShard is every +//! document and index entry of its collections: storage key and stored +//! bytes, in key order. Two replicas that installed the same entries in the +//! same order hold equal content. + +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use nodedb::types::{DatabaseId, TenantId}; + +use crate::common::cluster_harness::shared_steps::{db_detail, key_collection, name_where}; +use crate::common::cluster_harness::{TestCluster, TestClusterNode, is_no_serving_leader}; + +/// The tenant the `nodedb` pgwire user writes as. +const TENANT: u64 = 1; + +/// SQLSTATE of a transaction aborted by a conflict. It wrote nothing. +const SERIALIZATION_FAILURE: &str = "40001"; + +/// How long a refused statement is retried. +const RETRY_DEADLINE: Duration = Duration::from_secs(30); + +/// Pause between two attempts of a refused statement. +const RETRY_BACKOFF: Duration = Duration::from_millis(50); + +/// How long the replicas get to agree after the cluster quiesced. +const AGREE_DEADLINE: Duration = Duration::from_secs(30); + +/// Pause between two content reads while the replicas disagree. +const AGREE_POLL: Duration = Duration::from_millis(200); + +/// The most differing keys a disagreement report names per vShard. +const REPORT_KEYS: usize = 10; + +/// One vShard's stored content on one replica, keyed by table and storage +/// key. +pub(super) type VShardContent = BTreeMap>; + +/// The vShard `collection` in the default database homes to. +pub(super) fn vshard_of(collection: &str) -> u32 { + nodedb_cluster::routing::vshard_for_collection(nodedb_types::CollectionKey::from_bare( + DatabaseId::DEFAULT, + collection, + )) +} + +/// One collection name per prefix, `{prefix}_{i}` for the lowest `i` whose +/// vShard differs from the vShard of every name picked before it. +pub(super) fn names_on_distinct_vshards(prefixes: [&str; N]) -> [String; N] { + let mut taken: Vec = Vec::with_capacity(N); + prefixes.map(|prefix| { + let name = name_where(prefix, |candidate| !taken.contains(&vshard_of(candidate))); + taken.push(vshard_of(&name)); + name + }) +} + +/// A fresh session on `node` in strict cross-shard mode. +pub(super) async fn strict_session(node: &TestClusterNode) -> tokio_postgres::Client { + let (client, connection) = tokio_postgres::connect( + &format!( + "host=127.0.0.1 port={} user=nodedb dbname=default", + node.pg_addr.port() + ), + tokio_postgres::NoTls, + ) + .await + .expect("connect a session"); + tokio::spawn(async move { + let _ = connection.await; + }); + client + .simple_query("SET cross_shard_txn = 'strict'") + .await + .expect("SET cross_shard_txn = strict"); + client +} + +/// Run `sql` through `client` as one request. A `40001` refusal aborted the +/// transaction and wrote nothing, and a missing leader refused it before it +/// ran: either runs again until [`RETRY_DEADLINE`]. Any other error panics +/// with `what`, the SQLSTATE and the detail. +pub(super) async fn run_retrying(client: &tokio_postgres::Client, what: &str, sql: &str) { + let deadline = tokio::time::Instant::now() + RETRY_DEADLINE; + loop { + let error = match client.simple_query(sql).await { + Ok(_) => return, + Err(error) => error, + }; + let retryable = is_no_serving_leader(&error) + || error + .as_db_error() + .is_some_and(|db| db.code().code() == SERIALIZATION_FAILURE); + if !retryable || tokio::time::Instant::now() >= deadline { + panic!("{what}: {}", db_detail(&error)); + } + // A failed block can leave the session in an aborted transaction. + // Outside one, ROLLBACK only warns, so its result does not matter. + let _ = client.simple_query("ROLLBACK").await; + tokio::time::sleep(RETRY_BACKOFF).await; + } +} + +/// Create `target` and `source` as strict collections and declare +/// `target.balance` as the materialized sum of `source.amount` over +/// `source.account_id = target.id`. +pub(super) fn sum_ddl(source: &str, target: &str) -> [String; 3] { + [ + format!( + "CREATE COLLECTION {target} (id TEXT PRIMARY KEY, owner TEXT) \ + WITH (engine='document_strict')" + ), + format!( + "CREATE COLLECTION {source} (id TEXT PRIMARY KEY, account_id TEXT, amount TEXT) \ + WITH (engine='document_strict')" + ), + format!( + "ALTER COLLECTION {target} ADD COLUMN balance TEXT \ + MATERIALIZED_SUM SOURCE {source} \ + ON {source}.account_id = {target}.id VALUE {source}.amount" + ), + ] +} + +/// The first column of every row `sql` returns through `client`. +pub(super) async fn first_column(client: &tokio_postgres::Client, sql: &str) -> Vec { + client + .simple_query(sql) + .await + .unwrap_or_else(|e| panic!("`{sql}`: {}", db_detail(&e))) + .into_iter() + .filter_map(|msg| match msg { + tokio_postgres::SimpleQueryMessage::Row(row) => row.get(0).map(str::to_owned), + _ => None, + }) + .collect() +} + +/// The content of each vShard `collections` home to, as `node` stores it. +pub(super) async fn local_content( + node: &TestClusterNode, + collections: &[&str], +) -> BTreeMap { + let mut by_vshard: BTreeMap = collections + .iter() + .map(|collection| (vshard_of(collection), VShardContent::new())) + .collect(); + for core in 0..node.num_cores() { + let snapshot = node + .tenant_snapshot_on_core(core, TenantId::new(TENANT)) + .await; + for (table, entries) in [("doc", snapshot.documents), ("idx", snapshot.indexes)] { + for (key, value) in entries { + let Some(collection) = key_collection(&key) else { + continue; + }; + if !collections.contains(&collection) { + continue; + } + let vshard = vshard_of(collection); + if let Some(content) = by_vshard.get_mut(&vshard) { + content.insert(format!("{table}:{key}"), value); + } + } + } + } + by_vshard +} + +/// The indexes in `cluster.nodes` of the replicas of `collection`'s data +/// group. Panics when fewer than two nodes replicate it: one replica agrees +/// with itself and proves nothing. +fn replicas_of(cluster: &TestCluster, collection: &str) -> Vec { + let replicas: Vec = cluster + .nodes + .iter() + .enumerate() + .filter(|(_, node)| { + node.group_id_for_collection(collection) + .is_some_and(|group| node.replicates_data_group(group)) + }) + .map(|(idx, _)| idx) + .collect(); + assert!( + replicas.len() >= 2, + "{collection}'s data group has {} replica(s); the comparison needs two or more", + replicas.len() + ); + replicas +} + +/// Wait until every replica of each vShard `collections` home to holds the +/// same content, and return that content per vShard. Panics at the deadline +/// with the node ids and the keys that differ. +pub(super) async fn wait_replicas_agree( + cluster: &TestCluster, + collections: &[&str], +) -> BTreeMap { + let replicas: BTreeMap> = collections + .iter() + .map(|collection| (vshard_of(collection), replicas_of(cluster, collection))) + .collect(); + let deadline = tokio::time::Instant::now() + AGREE_DEADLINE; + loop { + let mut per_node = Vec::with_capacity(cluster.nodes.len()); + for node in &cluster.nodes { + per_node.push(local_content(node, collections).await); + } + let disagreements: Vec = replicas + .iter() + .filter_map(|(vshard, nodes)| disagreement(cluster, &per_node, *vshard, nodes)) + .collect(); + if disagreements.is_empty() { + return replicas + .iter() + .map(|(vshard, nodes)| { + let content = per_node[nodes[0]].get(vshard).cloned().unwrap_or_default(); + (*vshard, content) + }) + .collect(); + } + if tokio::time::Instant::now() >= deadline { + panic!( + "replicas still hold different content after quiesce:\n{}", + disagreements.join("\n") + ); + } + tokio::time::sleep(AGREE_POLL).await; + } +} + +/// A report of how `vshard`'s replicas `nodes` differ, `None` when they +/// agree. A key is listed when one replica lacks it or stores other bytes. +fn disagreement( + cluster: &TestCluster, + per_node: &[BTreeMap], + vshard: u32, + nodes: &[usize], +) -> Option { + let empty = VShardContent::new(); + let content = |idx: usize| per_node[idx].get(&vshard).unwrap_or(&empty); + let reference = content(nodes[0]); + let reference_id = cluster.nodes[nodes[0]].node_id; + let differing: Vec = nodes[1..] + .iter() + .filter(|idx| content(**idx) != reference) + .flat_map(|idx| { + let other = content(*idx); + let other_id = cluster.nodes[*idx].node_id; + let keys: BTreeSet<&String> = reference.keys().chain(other.keys()).collect(); + keys.into_iter() + .filter(|key| reference.get(*key) != other.get(*key)) + .map(|key| format!("node {reference_id} vs node {other_id}: {key}")) + .collect::>() + }) + .take(REPORT_KEYS) + .collect(); + (!differing.is_empty()).then(|| { + let sizes: Vec = nodes + .iter() + .map(|idx| { + format!( + "node {}: {} entries", + cluster.nodes[*idx].node_id, + content(*idx).len() + ) + }) + .collect(); + format!( + "vShard {vshard} ({}): {}", + sizes.join(", "), + differing.join("; ") + ) + }) +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_convergence.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_convergence.rs new file mode 100644 index 000000000..e2978d72f --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_convergence.rs @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Concurrent fast-path writes and cross-shard Calvin transactions on the +//! same rows leave every replica of every vShard with the same content. +//! +//! A 3-node cluster holds four collections, each on its own vShard: +//! - `accts`: strict, its `balance` is the materialized sum of +//! `entries.amount` per account, +//! - `entries`: strict, the sum's source, +//! - `notes` and `scratch`: keyed. +//! +//! Three sessions on three nodes run at once: +//! - fast path: single-shard autocommit writes to `accts`, `notes` and +//! `scratch`, +//! - Calvin: transactions that insert and delete `entries` rows and update +//! the same `notes` rows, +//! - Calvin with TRUNCATE: transactions that truncate `scratch`, and once +//! `entries`, next to a `notes` insert. +//! +//! After quiesce, each vShard's documents and index entries are byte-equal +//! on every replica, read from each node's own cores. Each account's served +//! balance equals the sum of its served entries. + +use super::calvin_multishard_fixture::{Fixture, keyed_ddl}; +use super::calvin_replica_content::{ + first_column, names_on_distinct_vshards, run_retrying, strict_session, sum_ddl, vshard_of, + wait_replicas_agree, +}; + +/// Rounds each session runs. +const ROUNDS: usize = 20; + +/// Accounts in `accts`, `acc-0` to `acc-{ACCOUNTS - 1}`. +const ACCOUNTS: usize = 3; + +/// Seeded `notes` rows, `n0` to `n{NOTES - 1}`. +const NOTES: usize = 5; + +/// The round whose TRUNCATE names `entries`. Every other round truncates +/// `scratch`. +const ENTRIES_TRUNCATE_ROUND: usize = ROUNDS / 2; + +/// The four collections of the case. +struct Names { + accts: String, + entries: String, + notes: String, + scratch: String, +} + +impl Names { + fn all(&self) -> [&str; 4] { + [ + self.accts.as_str(), + self.entries.as_str(), + self.notes.as_str(), + self.scratch.as_str(), + ] + } +} + +/// Single-shard autocommit writes: an `accts` column the sum leaves alone, +/// a `notes` row the Calvin session also updates, and a new `scratch` row. +async fn fast_path_writes(client: &tokio_postgres::Client, names: &Names) { + for k in 0..ROUNDS { + let account = k % ACCOUNTS; + let note = k % NOTES; + let statements = [ + format!( + "UPDATE {} SET owner = 'fp{k}' WHERE id = 'acc-{account}'", + names.accts + ), + format!( + "UPDATE {} SET v = 'fp{k}' WHERE id = 'n{note}'", + names.notes + ), + format!( + "INSERT INTO {} (id, v) VALUES ('fp{k}', 'fp')", + names.scratch + ), + ]; + for sql in &statements { + run_retrying(client, &format!("fast path round {k}"), sql).await; + } + } +} + +/// Cross-shard transactions: each inserts an entry, which moves its +/// account's balance on another vShard, and updates a `notes` row. Every +/// third round also deletes an earlier entry. +async fn calvin_writes(client: &tokio_postgres::Client, names: &Names) { + for k in 0..ROUNDS { + let account = k % ACCOUNTS; + let note = k % NOTES; + let amount = k + 1; + let delete = if k >= 3 && k % 3 == 0 { + format!("DELETE FROM {} WHERE id = 'e{}'; ", names.entries, k - 3) + } else { + String::new() + }; + let txn = format!( + "BEGIN; \ + INSERT INTO {entries} (id, account_id, amount) \ + VALUES ('e{k}', 'acc-{account}', '{amount}'); \ + UPDATE {notes} SET v = 'cv{k}' WHERE id = 'n{note}'; \ + {delete}\ + COMMIT", + entries = names.entries, + notes = names.notes, + ); + run_retrying(client, &format!("calvin round {k}"), &txn).await; + } +} + +/// Cross-shard transactions that truncate a collection and insert a +/// `notes` row. Round [`ENTRIES_TRUNCATE_ROUND`] truncates `entries`, which +/// takes every truncated entry back off its account's balance. +async fn truncating_writes(client: &tokio_postgres::Client, names: &Names) { + for k in 0..ROUNDS { + let truncated = if k == ENTRIES_TRUNCATE_ROUND { + &names.entries + } else { + &names.scratch + }; + let txn = format!( + "BEGIN; \ + TRUNCATE {truncated}; \ + INSERT INTO {notes} (id, v) VALUES ('t{k}', 'tr'); \ + COMMIT", + notes = names.notes, + ); + run_retrying(client, &format!("truncate round {k}"), &txn).await; + } +} + +/// A served number column as `f64`. +fn number(text: &str, what: &str) -> f64 { + text.parse() + .unwrap_or_else(|e| panic!("{what} `{text}` is not a number: {e}")) +} + +/// Each account's served balance equals the sum of its served entries. +async fn assert_balances_match_entries(client: &tokio_postgres::Client, names: &Names) { + for account in 0..ACCOUNTS { + let balances = first_column( + client, + &format!( + "SELECT balance FROM {} WHERE id = 'acc-{account}'", + names.accts + ), + ) + .await; + let [balance] = balances.as_slice() else { + panic!("acc-{account} has one row, found balances {balances:?}"); + }; + let amounts = first_column( + client, + &format!( + "SELECT amount FROM {} WHERE account_id = 'acc-{account}'", + names.entries + ), + ) + .await; + let expected: f64 = amounts.iter().map(|a| number(a, "amount")).sum(); + assert_eq!( + number(balance, "balance"), + expected, + "acc-{account}'s balance is the sum of its entries {amounts:?}" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn replicas_converge_under_concurrent_fast_path_and_calvin_writes() { + let [accts, entries, notes, scratch] = + names_on_distinct_vshards(["conv_accts", "conv_entries", "conv_notes", "conv_scratch"]); + let names = Names { + accts, + entries, + notes, + scratch, + }; + let [accts_ddl, entries_ddl, sum] = sum_ddl(&names.entries, &names.accts); + let fx = Fixture::spawn(&[ + accts_ddl, + entries_ddl, + keyed_ddl(&names.notes), + keyed_ddl(&names.scratch), + ]) + .await; + fx.cluster + .exec_ddl_on_any_leader(&sum) + .await + .unwrap_or_else(|e| panic!("{sum}: {e}")); + for collection in names.all() { + fx.wait_group_mounted(collection).await; + } + + let seeder = &fx.cluster.nodes[0].client; + for account in 0..ACCOUNTS { + run_retrying( + seeder, + "seed an account", + &format!( + "INSERT INTO {} (id, owner, balance) VALUES ('acc-{account}', 'seed', '0')", + names.accts + ), + ) + .await; + } + for note in 0..NOTES { + run_retrying( + seeder, + "seed a note", + &format!( + "INSERT INTO {} (id, v) VALUES ('n{note}', 'seed')", + names.notes + ), + ) + .await; + } + fx.converge().await; + + let calvin = strict_session(&fx.cluster.nodes[1]).await; + let truncating = strict_session(&fx.cluster.nodes[2]).await; + tokio::join!( + fast_path_writes(&fx.cluster.nodes[0].client, &names), + calvin_writes(&calvin, &names), + truncating_writes(&truncating, &names), + ); + + fx.converge().await; + let content = wait_replicas_agree(&fx.cluster, &names.all()).await; + // The `entries` truncate can run after the last insert, so only the + // seeded collections are sure to hold rows. + for collection in [&names.accts, &names.notes] { + assert!( + content + .get(&vshard_of(collection)) + .is_some_and(|stored| !stored.is_empty()), + "{collection} holds rows after the workload; equal empty replicas prove nothing" + ); + } + assert_balances_match_entries(&fx.coordinator().client, &names).await; + + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs index 925c17c4d..3136d886f 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs @@ -13,6 +13,7 @@ mod backup_schedule_failover; mod bitemporal_array_cluster; mod calvin_cdc_net_kinds; mod calvin_cluster_pgwire_e2e; +mod calvin_duplicate_redo_copy; mod calvin_joiner_after_sequencer_compaction; mod calvin_leader_change_mid_txn; mod calvin_multi_shard_bitemporal_best_effort_restart; @@ -26,6 +27,8 @@ mod calvin_ollp_cross_node; mod calvin_ollp_pk_delete; mod calvin_ollp_update; mod calvin_remote_participant_report; +mod calvin_replica_content; +mod calvin_replica_convergence; mod calvin_submit_routed_cross_node; mod calvin_superseded_collection_cluster; mod catalog_put_if_absent; diff --git a/nodedb-physical/src/physical_plan/document/sum_target.rs b/nodedb-physical/src/physical_plan/document/sum_target.rs index bb619a42d..1824eecd7 100644 --- a/nodedb-physical/src/physical_plan/document/sum_target.rs +++ b/nodedb-physical/src/physical_plan/document/sum_target.rs @@ -129,6 +129,11 @@ impl ResolvedSumTarget { /// A transaction redo record carries source post-images only. Every replica /// applying it folds the source rows into their target rows, so the target /// identities travel with the redo, keyed by source collection. +/// +/// The table holds only the targets the source rows fold on their own core. +/// A target whose balance travels on an `ApplyBalanceDelta` task has no entry +/// here. The install then skips it. The table is merged over every write of +/// the collection, so an entry here applies to every row of the collection. #[derive( Debug, Clone, @@ -142,11 +147,8 @@ impl ResolvedSumTarget { pub struct RedoSumTargets { /// SOURCE collection the transaction wrote. pub collection: String, - /// Every target row the transaction's writes to `collection` resolved. + /// Every target row the transaction's writes to `collection` fold into. pub resolved: Vec, - /// TARGET collections whose delta travels on its own `ApplyBalanceDelta` - /// task, so the fold skips them. - pub deferred: Vec, } /// The surrogate `resolved` binds `target_collection`'s `join_value` to. diff --git a/nodedb-test-support/src/cluster_harness/node/inspect/core_placement.rs b/nodedb-test-support/src/cluster_harness/node/inspect/core_placement.rs index cdaa555ad..888661475 100644 --- a/nodedb-test-support/src/cluster_harness/node/inspect/core_placement.rs +++ b/nodedb-test-support/src/cluster_harness/node/inspect/core_placement.rs @@ -42,6 +42,23 @@ impl TestClusterNode { /// The document keys of `tenant` in the default database that core /// `core_id` stores, from that core's own tenant snapshot. pub async fn document_keys_on_core(&self, core_id: usize, tenant: TenantId) -> Vec { + self.tenant_snapshot_on_core(core_id, tenant) + .await + .documents + .into_iter() + .map(|(k, _)| k) + .collect() + } + + /// The tenant snapshot of `tenant` in the default database that core + /// `core_id` of this node holds. The read never leaves this node, so it + /// shows this replica's own stored rows. Panics when the core answers + /// with an error. + pub async fn tenant_snapshot_on_core( + &self, + core_id: usize, + tenant: TenantId, + ) -> TenantDataSnapshot { let request_id = RequestId::new(PLACEMENT_REQUEST_ID.fetch_add(1, Ordering::Relaxed)); let request = Request { request_id, @@ -87,8 +104,6 @@ impl TestClusterNode { Status::Ok, "core {core_id} snapshot failed" ); - let snap: TenantDataSnapshot = - zerompk::from_msgpack(response.payload.as_bytes()).expect("core snapshot decodes"); - snap.documents.into_iter().map(|(k, _)| k).collect() + zerompk::from_msgpack(response.payload.as_bytes()).expect("core snapshot decodes") } } diff --git a/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs b/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs index e80324678..748ab9430 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs @@ -117,6 +117,11 @@ fn claim( ledger, }), Err(refusal) => { + state + .calvin + .counters + .redo_copies_skipped + .fetch_add(1, Ordering::Relaxed); tracing::debug!( vshard_id, epoch, @@ -211,6 +216,62 @@ impl CalvinClaim { }; inbox.push(self.epoch, self.position, event).await; } + + /// Hand an install's `event` to the vShard's scheduler. `collections` + /// are the slice's collections. + /// + /// With fail points, the gate `calvin::before_redo_applied_push::` on + /// one of them withholds a `RedoApplied`. The entry concludes, and its + /// own task pushes the event once the gate releases. Until then the + /// leader sees its group apply past the redo with no install heard, and + /// proposes the redo again. + pub(super) async fn report_install( + &self, + state: &SharedState, + event: CalvinApplyEvent, + collections: &[String], + ) { + #[cfg(feature = "failpoints")] + let Some(event) = self.withhold_applied(state, event, collections) else { + return; + }; + #[cfg(not(feature = "failpoints"))] + let _ = collections; + self.report(state, event).await; + } + + /// Hand a held `RedoApplied` to a task that pushes it once its gate + /// releases. Returns any other event, or one no gate holds. + #[cfg(feature = "failpoints")] + fn withhold_applied( + &self, + state: &SharedState, + event: CalvinApplyEvent, + collections: &[String], + ) -> Option { + if !matches!(event, CalvinApplyEvent::RedoApplied { .. }) { + return Some(event); + } + let Some(gate) = crate::control::fail_gate::holds_redo_applied_push(collections) else { + return Some(event); + }; + let Some(inbox) = state.calvin.inboxes.get(self.vshard_id) else { + return Some(event); + }; + let (epoch, position) = (self.epoch, self.position); + tracing::info!( + vshard_id = self.vshard_id, + epoch, + position, + gate = %gate, + "calvin: RedoApplied withheld at a fail point" + ); + tokio::spawn(async move { + crate::control::fail_gate::released(&gate).await; + inbox.push(epoch, position, event).await; + }); + None + } } #[cfg(test)] diff --git a/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs b/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs index 094fda2fd..34e59d9de 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/transaction_redo.rs @@ -297,7 +297,9 @@ pub(super) fn enqueue_payload<'a>( // so before the group's applied index passes it. if let Some(claim) = &calvin { let event = claim.settle(state, &submitted); - claim.report(state, event).await; + claim + .report_install(state, event, &payload.collections) + .await; } let outcome = conclude_transaction_redo(tracker, pos, submitted); settle_stream(ctx, stream, &outcome); diff --git a/nodedb/src/control/fail_gate.rs b/nodedb/src/control/fail_gate.rs index 6bd860104..6cea112b0 100644 --- a/nodedb/src/control/fail_gate.rs +++ b/nodedb/src/control/fail_gate.rs @@ -83,24 +83,57 @@ pub(crate) async fn after_calvin_stamp(tx_class: &nodedb_cluster::calvin::types: /// tick. On its first hold the gate creates `.parked`, so a test can /// wait until the slice is held. pub(crate) fn holds_redo_propose(collections: &[String]) -> bool { - collections.iter().any(|collection| { - let name = format!("calvin::before_redo_propose::{collection}"); - let Some(FailAction::WaitForFile(path)) = lookup(&name) else { - return false; - }; - if path.exists() { - return false; - } - let mut parked = path.into_os_string(); - parked.push(".parked"); - let parked = std::path::PathBuf::from(parked); - if !parked.exists() - && let Err(e) = std::fs::write(&parked, &name) - { - tracing::warn!(gate = %name, error = %e, "fail gate parked marker not created"); - } - true - }) + collections + .iter() + .any(|collection| holds(&format!("calvin::before_redo_propose::{collection}"))) +} + +/// The Calvin install gate: after a stamped redo's install is durable and +/// before its `RedoApplied` event reaches the vShard's scheduler. Named per +/// collection, `calvin::before_redo_applied_push::`. +/// +/// The apply loop cannot park here without holding the entry back. Returns +/// the name of the gate that holds one of `collections`: its armed file is +/// absent. The caller hands the event to its own task, which waits on +/// [`released`]. On its first hold the gate creates `.parked`. +pub(crate) fn holds_redo_applied_push(collections: &[String]) -> Option { + collections + .iter() + .map(|collection| format!("calvin::before_redo_applied_push::{collection}")) + .find(|name| holds(name)) +} + +/// Resolves once gate `name` stops holding: its armed file exists, or +/// nothing is armed for it. +pub(crate) async fn released(name: &str) { + while armed_and_absent(name).is_some() { + tokio::time::sleep(GATE_POLL).await; + } +} + +/// The file armed for `name` with `wait_file()`, while it is absent. +fn armed_and_absent(name: &str) -> Option { + let Some(FailAction::WaitForFile(path)) = lookup(name) else { + return None; + }; + (!path.exists()).then_some(path) +} + +/// Whether gate `name` holds now: its armed file is absent. On its first +/// hold it creates `.parked`. +fn holds(name: &str) -> bool { + let Some(path) = armed_and_absent(name) else { + return false; + }; + let mut parked = path.into_os_string(); + parked.push(".parked"); + let parked = std::path::PathBuf::from(parked); + if !parked.exists() + && let Err(e) = std::fs::write(&parked, name) + { + tracing::warn!(gate = %name, error = %e, "fail gate parked marker not created"); + } + true } /// The committed-message delivery gates of node `node_id`: @@ -151,20 +184,5 @@ async fn delivery_gate(name: &str, mut shutdown: tokio::sync::watch::Receiver.parked`, so a test can wait /// until the node's apply is held. pub(crate) fn holds_metadata_apply(node_id: u64) -> bool { - let name = crate::control::cluster::metadata_applier::metadata_apply_hold_point(node_id); - let Some(FailAction::WaitForFile(path)) = lookup(&name) else { - return false; - }; - if path.exists() { - return false; - } - let mut parked = path.into_os_string(); - parked.push(".parked"); - let parked = std::path::PathBuf::from(parked); - if !parked.exists() - && let Err(e) = std::fs::write(&parked, &name) - { - tracing::warn!(gate = %name, error = %e, "fail gate parked marker not created"); - } - true + holds(&crate::control::cluster::metadata_applier::metadata_apply_hold_point(node_id)) } diff --git a/nodedb/src/control/server/native/dispatch/sql_fold.rs b/nodedb/src/control/server/native/dispatch/sql_fold.rs index 9852f8b61..413f6faf8 100644 --- a/nodedb/src/control/server/native/dispatch/sql_fold.rs +++ b/nodedb/src/control/server/native/dispatch/sql_fold.rs @@ -17,6 +17,7 @@ use crate::control::server::response_shape::types::{ }; use crate::control::server::shared::session::staging_gate::{StagedTagKind, StagedWriteOutcome}; use crate::types::DatabaseId; +use nodedb_physical::physical_task::PhysicalTask; use super::{ DispatchCtx, apply_dml_outcome, dml_fold_error_to_native, shape_error_to_native, @@ -35,7 +36,6 @@ pub(super) struct FoldShape<'a> { } /// One statement's response, accumulated across its tasks. -#[derive(Default)] pub(super) struct StatementFold { pub columns: Option>, pub rows: Vec>, @@ -45,14 +45,29 @@ pub(super) struct StatementFold { } impl StatementFold { - /// Fold one task's count-bearing outcome into the statement's tag. + /// An empty fold for the statement that planned `tasks`. The tag is built + /// over every task, so a derived write beside the user's own adds no + /// verb and no count. + pub(super) fn for_tasks(tasks: &[PhysicalTask]) -> Self { + Self { + columns: None, + rows: Vec::new(), + warnings: Vec::new(), + tag: StatementTag::for_plans(tasks.iter().map(|t| &t.plan)), + last_lsn: 0, + } + } + + /// Fold the count-bearing outcome of the task that ran `plan` into the + /// statement's tag. pub(super) fn fold_dml( &mut self, seq: u64, + plan: &PhysicalPlan, outcome: DmlOutcome, ) -> Result<(), Box> { self.tag - .fold(outcome) + .fold(self.tag.role_of(plan), outcome) .map_err(|e| Box::new(dml_fold_error_to_native(seq, &e))) } @@ -114,6 +129,7 @@ impl StatementFold { if !matches!(outcome.kind, StagedTagKind::RawPayload) { return self.fold_dml( shape.seq, + plan, staged_dml_outcome(outcome.kind, outcome.affected), ); } diff --git a/nodedb/src/control/server/native/dispatch/sql_loop.rs b/nodedb/src/control/server/native/dispatch/sql_loop.rs index 742697c10..fbd0c2ffc 100644 --- a/nodedb/src/control/server/native/dispatch/sql_loop.rs +++ b/nodedb/src/control/server/native/dispatch/sql_loop.rs @@ -16,13 +16,10 @@ use std::sync::Arc; use nodedb_types::protocol::NativeResponse; use crate::bridge::envelope::Status; -use crate::control::planner::calvin::write_class::{ - plan_counts_toward_statement_tag, plans_have_user_write, -}; use crate::control::sequence::SessionSequenceAccess; use crate::control::server::response_shape::schema::OutputSchema; use crate::control::server::response_shape::types::{ - PlanKind, describe_plan, payload_to_dml_outcome, replaced_write_outcome, + PlanKind, TaskTagRole, describe_plan, payload_to_dml_outcome, replaced_write_outcome, }; use crate::control::server::shared::metering::{PlanMeteringInfo, meter_dispatch}; use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; @@ -92,15 +89,15 @@ pub(super) async fn run_dispatch_loop( txn.sessions .record_read_entries(txn.session_id, sum_target_reads); } - let mut fold = StatementFold::default(); + // A derived write beside the user's own (an implicit edge, a cross-shard + // balance move) never answers the statement, exactly as Calvin's deposit + // rule has it: the fold is built over every task to decide it. + let mut fold = StatementFold::for_tasks(&tasks); // Checked once rather than per task — metering is disabled by default, so // this keeps the per-task extraction below (which clones the collection // name) a true no-op on the hot path for every deployment that hasn't // turned it on. let metering_enabled = ctx.state.metering_config.enabled; - // A derived implicit-edge write beside the user's own never answers the - // statement, exactly as Calvin's deposit rule has it. - let has_user_write = plans_have_user_write(tasks.iter().map(|t| &t.plan)); // Session-scoped sequence access for the statement's Control-Plane // computed columns: the registry plus this connection's `currval` map. let session_sequences = ctx.sessions.sequence_values(ctx.peer_addr); @@ -174,19 +171,12 @@ pub(super) async fn run_dispatch_loop( ControlFlow::Break(fold.fold_staged(ctx, &shape, &plan, outcome)) } Ok(TxnTaskOutcome::CloneHandled(clone_resp)) => ControlFlow::Break( - fold_answer( - ctx, - &shape, - &mut fold, - &plan, - has_user_write, - clone_resp.payload.as_ref(), - ) - .map(drop), + fold_answer(ctx, &shape, &mut fold, &plan, clone_resp.payload.as_ref()) + .map(drop), ), Ok(TxnTaskOutcome::InsteadOf) => { ControlFlow::Break(match replaced_write_outcome(describe_plan(&plan)) { - Some(outcome) => fold.fold_dml(seq, outcome), + Some(outcome) => fold.fold_dml(seq, &plan, outcome), None => { fold.tag.fold_opaque(); Ok(()) @@ -242,7 +232,7 @@ pub(super) async fn run_dispatch_loop( fold.rows.extend(rows); } Ok(super::cluster_array::ClusterArrayOutcome::Affected(outcome)) => { - if let Err(e) = fold.fold_dml(seq, outcome) { + if let Err(e) = fold.fold_dml(seq, &plan, outcome) { return SqlOutcome::Response(e); } } @@ -310,14 +300,8 @@ pub(super) async fn run_dispatch_loop( fold.last_lsn = task_resp.watermark_lsn.as_u64(); - let task_rows = match fold_answer( - ctx, - &shape, - &mut fold, - &plan, - has_user_write, - task_resp.payload.as_ref(), - ) { + let task_rows = match fold_answer(ctx, &shape, &mut fold, &plan, task_resp.payload.as_ref()) + { Ok(rows) => rows, Err(e) => return SqlOutcome::Response(e), }; @@ -382,11 +366,9 @@ fn fold_answer( shape: &FoldShape<'_>, fold: &mut StatementFold, plan: &crate::bridge::envelope::PhysicalPlan, - has_user_write: bool, payload: &[u8], ) -> Result, Box> { let plan_kind = describe_plan(plan); - let counts_toward_tag = plan_counts_toward_statement_tag(plan, has_user_write); let count_bearing = matches!(plan_kind, PlanKind::DmlResult(_) | PlanKind::DmlResultByOp); if payload.is_empty() && !count_bearing { // Not a count-bearing plan (graph / vector / index write): no count @@ -402,14 +384,14 @@ fn fold_answer( // opaque execution reports neither. Counting one per dispatched task // instead will report a row for a delete that removed nothing and for an // `ON CONFLICT DO NOTHING` insert that skipped. - if !counts_toward_tag { + if fold.tag.role_of(plan) == TaskTagRole::Opaque { fold.tag.fold_opaque(); return Ok(None); } match payload_to_dml_outcome(payload, plan_kind) { Ok(Some(outcome)) => { let affected = outcome.affected; - fold.fold_dml(shape.seq, outcome)?; + fold.fold_dml(shape.seq, plan, outcome)?; Ok(Some(affected)) } Ok(None) => { diff --git a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs index 453266587..186d34270 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs @@ -15,13 +15,10 @@ use std::sync::Arc; use pgwire::api::results::Response; use pgwire::error::{ErrorInfo, PgWireError, PgWireResult}; -use crate::control::planner::calvin::write_class::{ - plan_counts_toward_statement_tag, plans_have_user_write, -}; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::security::request_scope::RequestAuthScope; use crate::control::server::response_shape::redaction::QueryRedaction; -use crate::control::server::response_shape::types::{ShapedRows, StatementTag}; +use crate::control::server::response_shape::types::{ShapedRows, StatementTag, TaskTagRole}; use crate::control::server::shared::ddl::neutral::maintenance::auto_analyze; use crate::control::server::shared::metering::{PlanMeteringInfo, meter_dispatch}; use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; @@ -104,8 +101,11 @@ impl NodeDbPgHandler { let mut returning_rows: Option = None; // The statement's ONE command tag, folded over its write tasks the // same way. `execute_sql` calls this loop once per `;`-separated - // statement, so the fold never crosses a statement boundary. - let mut statement_tag = StatementTag::default(); + // statement, so the fold never crosses a statement boundary. Built + // over every task, so a derived write beside the user's own (an + // implicit edge, a cross-shard balance move) adds no verb and no + // count, in a transaction or out of one. + let mut statement_tag = StatementTag::for_plans(tasks.iter().map(|t| &t.plan)); let mut responses = Vec::with_capacity(tasks.len()); // Session-scoped sequence access for the statement's Control-Plane // computed columns, resolved once: every task of one statement @@ -117,9 +117,6 @@ impl NodeDbPgHandler { // the collection name) a true no-op on the hot path for every // deployment that hasn't turned it on. let metering_enabled = self.state.metering_config.enabled; - // A derived implicit-edge write beside the user's own never answers - // the statement, exactly as Calvin's deposit rule has it. - let has_user_write = plans_have_user_write(tasks.iter().map(|t| &t.plan)); // A strong session reads linearizably, in and out of a transaction. let strong_reads = self.sessions.read_consistency(session_id).requires_leader(); @@ -156,6 +153,7 @@ impl NodeDbPgHandler { // rows into the statement's one result set. if let Some(route) = route_ctx.as_ref() { let plan = task.plan.clone(); + let tag_role = statement_tag.role_of(&plan); let task_database_id = task.database_id; let hooks = execute_dml_hooks::PreDispatchContext { identity, @@ -172,7 +170,7 @@ impl NodeDbPgHandler { { StatementTxnOutcome::Dispatch(routed_task) => task = *routed_task, StatementTxnOutcome::Write(handled) => { - handled.fold_into(&mut statement_tag)?; + handled.fold_into(&mut statement_tag, tag_role)?; continue; } StatementTxnOutcome::Returning(returning) => { @@ -182,7 +180,7 @@ impl NodeDbPgHandler { response: &staged_rows_response(rows), plan: &plan, plan_kind: PlanKind::ReturningRows, - counts_toward_tag: false, + tag_role: TaskTagRole::Opaque, projection, result_formats, session_id, @@ -202,7 +200,7 @@ impl NodeDbPgHandler { match self.clone_write_answer(hooks, &plan, &resp)? { PreDispatchHandled::Rows(response) => responses.push(response), PreDispatchHandled::Write(handled) => { - handled.fold_into(&mut statement_tag)?; + handled.fold_into(&mut statement_tag, tag_role)?; } } continue; @@ -245,14 +243,14 @@ impl NodeDbPgHandler { { ClusterArrayResult::Rows(response) => responses.push(response), ClusterArrayResult::Dml(outcome) => statement_tag - .fold(outcome) + .fold(statement_tag.role_of(&task.plan), outcome) .map_err(|e| dml_fold_error_to_pg(&e))?, } continue; } let plan_kind = describe_plan(&task.plan); - let counts_toward_tag = plan_counts_toward_statement_tag(&task.plan, has_user_write); + let tag_role = statement_tag.role_of(&task.plan); let resp_post_set_op = task.post_set_op; let task_database_id = task.database_id; let task_vshard = task.vshard_id; @@ -332,7 +330,7 @@ impl NodeDbPgHandler { execute_dml_hooks::PreDispatchOutcome::Handled(PreDispatchHandled::Write( handled, )) => { - handled.fold_into(&mut statement_tag)?; + handled.fold_into(&mut statement_tag, tag_role)?; continue; } execute_dml_hooks::PreDispatchOutcome::Proceed(proceed) => { @@ -425,7 +423,7 @@ impl NodeDbPgHandler { response: &resp, plan: &plan_for_response, plan_kind, - counts_toward_tag, + tag_role, projection, result_formats, session_id, diff --git a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/task.rs b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/task.rs index 8b9c925c7..b3189e979 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/task.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/task.rs @@ -15,7 +15,7 @@ use crate::control::server::response_shape::redaction::QueryRedaction; use crate::control::server::response_shape::request::MaterializedShapeRequest; use crate::control::server::response_shape::schema::OutputSchema; use crate::control::server::response_shape::types::{ - ShapedRows, StatementTag, payload_to_dml_outcome, + ShapedRows, StatementTag, TaskTagRole, payload_to_dml_outcome, }; use crate::control::server::shared::session::SessionId; use crate::types::{DatabaseId, TenantId}; @@ -30,9 +30,9 @@ pub(super) struct ShapeTaskParams<'a> { pub(super) response: &'a crate::bridge::envelope::Response, pub(super) plan: &'a PhysicalPlan, pub(super) plan_kind: PlanKind, - /// Whether this task's count answers the statement. False for a derived - /// implicit-edge write beside the user's own, which folds as opaque. - pub(super) counts_toward_tag: bool, + /// Whether this task's count answers the statement. `Opaque` for a + /// derived write beside the user's own, which folds as opaque. + pub(super) tag_role: TaskTagRole, pub(super) projection: Option<&'a OutputSchema>, pub(super) result_formats: &'a Format, pub(super) session_id: SessionId, @@ -64,7 +64,7 @@ impl NodeDbPgHandler { response, plan, plan_kind, - counts_toward_tag, + tag_role, projection, result_formats, session_id, @@ -113,7 +113,7 @@ impl NodeDbPgHandler { Ok(Some(task_rows)) } ShapeOutcome::Passthrough => { - if !counts_toward_tag { + if tag_role == TaskTagRole::Opaque { statement_tag.fold_opaque(); return Ok(None); } @@ -121,7 +121,7 @@ impl NodeDbPgHandler { .map_err(|e| error_to_pg(&e))? { Some(outcome) => statement_tag - .fold(outcome) + .fold(tag_role, outcome) .map_err(|e| dml_fold_error_to_pg(&e))?, None => statement_tag.fold_opaque(), } diff --git a/nodedb/src/control/server/pgwire/handler/routing/execute_dml_hooks.rs b/nodedb/src/control/server/pgwire/handler/routing/execute_dml_hooks.rs index 740afc556..8055e9256 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/execute_dml_hooks.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/execute_dml_hooks.rs @@ -14,7 +14,7 @@ use crate::control::security::auth_context::AuthContext; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::response_shape::schema::OutputSchema; use crate::control::server::response_shape::types::{ - DmlOutcome, StatementTag, payload_to_dml_outcome, + DmlOutcome, StatementTag, TaskTagRole, payload_to_dml_outcome, }; use crate::control::server::shared::session::SessionId; use crate::control::trigger::dml_hook::DmlWriteInfo; @@ -39,11 +39,16 @@ pub(super) enum HandledWrite { } impl HandledWrite { - /// Fold this contribution into the statement's tag. - pub(super) fn fold_into(self, statement_tag: &mut StatementTag) -> PgWireResult<()> { + /// Fold this contribution, made by a task whose role is `role`, into the + /// statement's tag. + pub(super) fn fold_into( + self, + statement_tag: &mut StatementTag, + role: TaskTagRole, + ) -> PgWireResult<()> { match self { HandledWrite::Dml(outcome) => statement_tag - .fold(outcome) + .fold(role, outcome) .map_err(|e| dml_fold_error_to_pg(&e)), HandledWrite::Opaque => { statement_tag.fold_opaque(); diff --git a/nodedb/src/control/server/pgwire/handler/routing/gateway_dispatch.rs b/nodedb/src/control/server/pgwire/handler/routing/gateway_dispatch.rs index a5e3c6e91..de326a381 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/gateway_dispatch.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/gateway_dispatch.rs @@ -13,9 +13,6 @@ use pgwire::api::results::Response; use pgwire::error::{ErrorInfo, PgWireError, PgWireResult}; use crate::control::gateway::GatewayErrorMap; -use crate::control::planner::calvin::write_class::{ - plan_counts_toward_statement_tag, plans_have_user_write, -}; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::security::request_scope::RequestAuthScope; use crate::control::server::response_shape::redaction::QueryRedaction; @@ -128,16 +125,15 @@ impl NodeDbPgHandler { linearizable: self.sessions.read_consistency(session_id).requires_leader(), }; - // A derived implicit-edge write beside the user's own never answers - // the statement, exactly as Calvin's deposit rule has it. - let has_user_write = plans_have_user_write(tasks.iter().map(|t| &t.plan)); - let mut fold = GatewayFold::with_capacity(tasks.len()); + // A derived write beside the user's own never answers the statement, + // exactly as Calvin's deposit rule has it. + let mut fold = GatewayFold::for_tasks(&tasks); for task in tasks { let plan_kind = describe_plan(&task.plan); // The task moves into authorization below; a row-producing task // keeps its plan for shaping the rows it answers with. let shape_plan = plan_produces_rows(plan_kind).then(|| task.plan.clone()); - let counts_toward_tag = plan_counts_toward_statement_tag(&task.plan, has_user_write); + let tag_role = fold.role_of(&task.plan); let metering_info = PlanMeteringInfo::extract(&task.plan); let emitter = crate::control::security::audit::ArcAuditEmitter(std::sync::Arc::clone( &self.state.audit, @@ -175,7 +171,7 @@ impl NodeDbPgHandler { resp.payload.as_ref(), plan_kind, shape_plan.as_ref(), - counts_toward_tag, + tag_role, &shaping, )?; meter_gateway_task( @@ -213,7 +209,7 @@ impl NodeDbPgHandler { &[], plan_kind, shape_plan.as_ref(), - counts_toward_tag, + tag_role, &shaping, )?; } @@ -223,7 +219,7 @@ impl NodeDbPgHandler { payload, plan_kind, shape_plan.as_ref(), - counts_toward_tag, + tag_role, &shaping, )? { task_rows = Some(task_rows.unwrap_or(0) + rows); diff --git a/nodedb/src/control/server/pgwire/handler/routing/gateway_fold.rs b/nodedb/src/control/server/pgwire/handler/routing/gateway_fold.rs index 86619e320..94edbe118 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/gateway_fold.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/gateway_fold.rs @@ -15,9 +15,10 @@ use crate::control::server::response_shape::redaction::QueryRedaction; use crate::control::server::response_shape::request::MaterializedShapeRequest; use crate::control::server::response_shape::schema::OutputSchema; use crate::control::server::response_shape::types::{ - ShapedRows, StatementTag, payload_to_dml_outcome, + ShapedRows, StatementTag, TaskTagRole, payload_to_dml_outcome, }; use crate::control::server::shared::session::SessionId; +use nodedb_physical::physical_task::PhysicalTask; use super::super::super::command_tag::push_folded_tag; use super::super::super::types::{dml_fold_error_to_pg, error_to_pg, shape_error_to_pg}; @@ -47,13 +48,19 @@ pub(super) struct GatewayFold { } impl GatewayFold { - pub(super) fn with_capacity(tasks: usize) -> Self { + /// An empty fold for the statement that planned `tasks`. + pub(super) fn for_tasks(tasks: &[PhysicalTask]) -> Self { Self { - responses: Vec::with_capacity(tasks), + responses: Vec::with_capacity(tasks.len()), returning_rows: None, - statement_tag: StatementTag::default(), + statement_tag: StatementTag::for_plans(tasks.iter().map(|t| &t.plan)), } } + + /// The role of the task that runs `plan` in this statement. + pub(super) fn role_of(&self, plan: &PhysicalPlan) -> TaskTagRole { + self.statement_tag.role_of(plan) + } } impl NodeDbPgHandler { @@ -87,8 +94,8 @@ impl NodeDbPgHandler { /// the fold, `RETURNING` rows accumulate into one result set, a /// passthrough payload folds its count and verb into the statement's /// tag, and an opaque execution folds as such. A task whose count does - /// not answer the statement (`counts_toward_tag` false: a derived - /// implicit-edge write beside the user's own) folds as opaque. + /// not answer the statement (`tag_role` is `Opaque`: a derived write + /// beside the user's own) folds as opaque. /// /// A row-producing payload shapes exactly as a locally dispatched one /// does, through `shape_response_materialized` with the task's plan. A @@ -107,7 +114,7 @@ impl NodeDbPgHandler { payload: &[u8], plan_kind: PlanKind, shape_plan: Option<&PhysicalPlan>, - counts_toward_tag: bool, + tag_role: TaskTagRole, shaping: &GatewayShaping<'_>, ) -> PgWireResult> { let outcome = match (plan_produces_rows(plan_kind), shape_plan) { @@ -154,7 +161,7 @@ impl NodeDbPgHandler { Ok(Some(rows)) } ShapeOutcome::Passthrough => { - if !counts_toward_tag { + if tag_role == TaskTagRole::Opaque { fold.statement_tag.fold_opaque(); return Ok(None); } @@ -163,7 +170,7 @@ impl NodeDbPgHandler { match payload_to_dml_outcome(payload, plan_kind).map_err(|e| error_to_pg(&e))? { Some(outcome) => fold .statement_tag - .fold(outcome) + .fold(tag_role, outcome) .map_err(|e| dml_fold_error_to_pg(&e))?, None => fold.statement_tag.fold_opaque(), } diff --git a/nodedb/src/control/server/response_shape/calvin_fold.rs b/nodedb/src/control/server/response_shape/calvin_fold.rs index 380d34b50..53947df71 100644 --- a/nodedb/src/control/server/response_shape/calvin_fold.rs +++ b/nodedb/src/control/server/response_shape/calvin_fold.rs @@ -15,13 +15,10 @@ use super::redaction::QueryRedaction; use super::request::MaterializedShapeRequest; use super::schema::OutputSchema; use super::types::{ - DmlFoldError, DmlOutcome, FoldedTag, PlanKind, ShapedRows, StatementTag, describe_plan, - dml_outcome_by_op, dml_outcome_from_payload, + DmlFoldError, DmlOutcome, FoldedTag, PlanKind, ShapedRows, StatementTag, TaskTagRole, + describe_plan, dml_outcome_by_op, dml_outcome_from_payload, }; use crate::bridge::envelope::Response; -use crate::control::planner::calvin::write_class::{ - plan_counts_toward_statement_tag, plans_have_user_write, -}; use crate::control::security::auth_context::AuthContext; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId}; @@ -78,11 +75,11 @@ pub fn fold_calvin_batch( apply_resp: Option<&Response>, ctx: &CalvinFoldCtx<'_>, ) -> Result { - let has_user_write = plans_have_user_write(plans.iter().copied()); let mut rows: Option = None; - let mut tag = StatementTag::default(); + let mut tag = StatementTag::for_plans(plans.iter().copied()); for plan in plans { - if !plan_counts_toward_statement_tag(plan, has_user_write) { + let role = tag.role_of(plan); + if role == TaskTagRole::Opaque { tag.fold_opaque(); continue; } @@ -90,7 +87,9 @@ pub fn fold_calvin_batch( CalvinTaskOutcome::Rows(shaped) => { rows.get_or_insert(shaped); } - CalvinTaskOutcome::Dml(outcome) => tag.fold(outcome).map_err(CalvinFoldError::Verb)?, + CalvinTaskOutcome::Dml(outcome) => { + tag.fold(role, outcome).map_err(CalvinFoldError::Verb)? + } CalvinTaskOutcome::Opaque => tag.fold_opaque(), } } diff --git a/nodedb/src/control/server/response_shape/types/dml_outcome.rs b/nodedb/src/control/server/response_shape/types/dml_outcome.rs index fbf32f7fb..6efa44556 100644 --- a/nodedb/src/control/server/response_shape/types/dml_outcome.rs +++ b/nodedb/src/control/server/response_shape/types/dml_outcome.rs @@ -7,6 +7,11 @@ //! The payload-to-outcome readers here are the one place a Data Plane //! count payload becomes a [`DmlOutcome`]; pgwire and native both call them. +use nodedb_physical::physical_plan::PhysicalPlan; + +use crate::control::planner::calvin::write_class::{ + plan_counts_toward_statement_tag, plans_have_user_write, +}; use crate::control::server::shared::sql::staging_predicates::{ StagedTagKind, extract_ingest_rejections, extract_kv_conflict_op, rejected_lines_notice, require_affected_count, @@ -57,10 +62,32 @@ pub enum DmlFoldError { /// - Any other verb mix: [`DmlFoldError::VerbMismatch`]. Never a silent pick. /// - An opaque task (`PlanKind::Execution`) adds nothing when a DML outcome /// is folded before or after it. Only-opaque folds render as `OK`. -#[derive(Debug, Default)] +/// - A task whose [`TaskTagRole`] is `Opaque` folds as an opaque task, whatever +/// outcome it reports. A derived write beside the user's own has that role: +/// a materialized-sum balance move or an implicit graph edge write. Its +/// count describes a row the statement never named. +/// +/// The tag is built over the statement's full plan set +/// ([`Self::for_plans`]), because one task alone cannot tell a derived write +/// beside the user's own from a statement that is only that write. +#[derive(Debug)] pub struct StatementTag { dml: Option, opaque: bool, + /// Whether the statement carries the user's own write. When it does, a + /// derived write folds as opaque. + has_user_write: bool, +} + +/// Whether one task's outcome answers the statement. Read from +/// [`StatementTag::role_of`] for the task's plan. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TaskTagRole { + /// The statement's own write: its verb and count fold into the tag. + Counts, + /// A task whose outcome adds no verb and no count: a derived write beside + /// the user's own, or a task whose rows answer in place of a count. + Opaque, } /// What a statement's fold produced. @@ -72,8 +99,32 @@ pub enum FoldedTag { } impl StatementTag { - /// Fold one task's count-bearing outcome into the statement's tag. - pub fn fold(&mut self, outcome: DmlOutcome) -> Result<(), DmlFoldError> { + /// An empty tag for the statement that planned `plans`: every task the + /// statement dispatches, stages or buffers. + pub fn for_plans<'a>(plans: impl IntoIterator) -> Self { + Self { + dml: None, + opaque: false, + has_user_write: plans_have_user_write(plans), + } + } + + /// The role of the task that runs `plan` in this statement. + pub fn role_of(&self, plan: &PhysicalPlan) -> TaskTagRole { + if plan_counts_toward_statement_tag(plan, self.has_user_write) { + TaskTagRole::Counts + } else { + TaskTagRole::Opaque + } + } + + /// Fold one task's count-bearing outcome into the statement's tag. A task + /// whose role is [`TaskTagRole::Opaque`] folds as opaque. + pub fn fold(&mut self, role: TaskTagRole, outcome: DmlOutcome) -> Result<(), DmlFoldError> { + if role == TaskTagRole::Opaque { + self.fold_opaque(); + return Ok(()); + } let Some(current) = self.dml else { self.dml = Some(outcome); return Ok(()); @@ -257,43 +308,116 @@ fn merged_verb(first: &'static str, second: &'static str) -> Option<&'static str mod tests { use super::*; + use nodedb_physical::physical_plan::{DocumentOp, GraphOp}; + use nodedb_types::{DatabaseId, QualifiedCollection, Surrogate}; + fn outcome(verb: &'static str, affected: u64) -> DmlOutcome { DmlOutcome { verb, affected } } + /// A tag for a statement with no plans: every task counts. + fn tag() -> StatementTag { + StatementTag::for_plans(std::iter::empty()) + } + + const COUNTS: TaskTagRole = TaskTagRole::Counts; + + fn entries() -> QualifiedCollection { + QualifiedCollection::new(DatabaseId::DEFAULT, "entries") + } + + /// The sum source's own insert. + fn source_insert() -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::PointInsert { + collection: entries(), + document_id: "e3".to_owned(), + value: Vec::new(), + if_absent: false, + surrogate: Surrogate::new(11), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + deferred_sum_targets: vec!["accts".to_owned()], + }) + } + + /// The sum source's own delete. + fn source_delete() -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::PointDelete { + collection: entries(), + document_id: "e0".into(), + surrogate: None, + pk_bytes: Vec::new(), + returning: None, + rls_filters: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::pending_injection(), + resolved_sum_targets: Vec::new(), + }) + } + + /// The cross-shard balance move the planner appends beside a sum-source + /// write. + fn balance_delta(delta: &str) -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::ApplyBalanceDelta { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "accts"), + document_id: "acc-0".to_owned(), + surrogate: Surrogate::new(7), + column: "balance".to_owned(), + delta: delta.to_owned(), + join_column: "account_id".to_owned(), + join_value: "acc-0".to_owned(), + declared_primary_key: None, + }) + } + + /// Fold each task's staged outcome in plan order, as the in-transaction + /// routes of both protocols do. + fn fold_staged(plans: &[(PhysicalPlan, StagedTagKind)]) -> Option { + let mut tag = StatementTag::for_plans(plans.iter().map(|(plan, _)| plan)); + for (plan, kind) in plans { + tag.fold(tag.role_of(plan), staged_dml_outcome(*kind, 1)) + .expect("a derived write never mixes its verb into the tag"); + } + tag.finish() + } + #[test] fn empty_fold_finishes_to_none() { - assert_eq!(StatementTag::default().finish(), None); + assert_eq!(tag().finish(), None); } #[test] fn same_verb_sums_affected() { - let mut tag = StatementTag::default(); - tag.fold(outcome("INSERT", 1)).expect("first fold"); - tag.fold(outcome("INSERT", 0)).expect("second fold"); - tag.fold(outcome("INSERT", 1)).expect("third fold"); + let mut tag = tag(); + tag.fold(COUNTS, outcome("INSERT", 1)).expect("first fold"); + tag.fold(COUNTS, outcome("INSERT", 0)).expect("second fold"); + tag.fold(COUNTS, outcome("INSERT", 1)).expect("third fold"); assert_eq!(tag.finish(), Some(FoldedTag::Dml(outcome("INSERT", 2)))); } #[test] fn insert_and_update_fold_to_insert_in_either_order() { - let mut tag = StatementTag::default(); - tag.fold(outcome("INSERT", 1)).expect("insert"); - tag.fold(outcome("UPDATE", 2)).expect("update after insert"); - assert_eq!(tag.finish(), Some(FoldedTag::Dml(outcome("INSERT", 3)))); - - let mut tag = StatementTag::default(); - tag.fold(outcome("UPDATE", 2)).expect("update"); - tag.fold(outcome("INSERT", 1)).expect("insert after update"); - assert_eq!(tag.finish(), Some(FoldedTag::Dml(outcome("INSERT", 3)))); + let mut first = tag(); + first.fold(COUNTS, outcome("INSERT", 1)).expect("insert"); + first + .fold(COUNTS, outcome("UPDATE", 2)) + .expect("update after insert"); + assert_eq!(first.finish(), Some(FoldedTag::Dml(outcome("INSERT", 3)))); + + let mut second = tag(); + second.fold(COUNTS, outcome("UPDATE", 2)).expect("update"); + second + .fold(COUNTS, outcome("INSERT", 1)) + .expect("insert after update"); + assert_eq!(second.finish(), Some(FoldedTag::Dml(outcome("INSERT", 3)))); } #[test] fn other_verb_mix_is_an_error() { - let mut tag = StatementTag::default(); - tag.fold(outcome("INSERT", 1)).expect("insert"); + let mut tag = tag(); + tag.fold(COUNTS, outcome("INSERT", 1)).expect("insert"); assert_eq!( - tag.fold(outcome("DELETE", 1)), + tag.fold(COUNTS, outcome("DELETE", 1)), Err(DmlFoldError::VerbMismatch { first: "INSERT", second: "DELETE", @@ -303,21 +427,85 @@ mod tests { #[test] fn opaque_never_changes_a_dml_tag() { - let mut tag = StatementTag::default(); + let mut tag = tag(); tag.fold_opaque(); - tag.fold(outcome("DELETE", 4)).expect("delete"); + tag.fold(COUNTS, outcome("DELETE", 4)).expect("delete"); tag.fold_opaque(); assert_eq!(tag.finish(), Some(FoldedTag::Dml(outcome("DELETE", 4)))); } #[test] fn only_opaque_finishes_to_opaque() { - let mut tag = StatementTag::default(); + let mut tag = tag(); tag.fold_opaque(); tag.fold_opaque(); assert_eq!(tag.finish(), Some(FoldedTag::Opaque)); } + /// An opaque-role task adds no verb and no count, whatever it reports. + #[test] + fn an_opaque_role_folds_as_opaque() { + let mut tag = tag(); + tag.fold(COUNTS, outcome("DELETE", 1)).expect("delete"); + tag.fold(TaskTagRole::Opaque, outcome("UPDATE", 1)) + .expect("an opaque role never mixes its verb"); + assert_eq!(tag.finish(), Some(FoldedTag::Dml(outcome("DELETE", 1)))); + } + + /// A DELETE of a sum-source row in a transaction block stages the delete + /// and the cross-shard balance move. The balance move is a derived + /// write, so the statement tags `DELETE 1`, never a verb mismatch. + #[test] + fn a_staged_sum_source_delete_tags_delete_1() { + let folded = fold_staged(&[ + (source_delete(), StagedTagKind::Delete), + (balance_delta("-4"), StagedTagKind::Update), + ]); + assert_eq!(folded, Some(FoldedTag::Dml(outcome("DELETE", 1)))); + } + + /// An INSERT of a sum-source row in a transaction block tags `INSERT 0 1`. + /// The balance move's row is not counted as an inserted row. + #[test] + fn a_staged_sum_source_insert_tags_insert_0_1() { + let folded = fold_staged(&[ + (source_insert(), StagedTagKind::Insert), + (balance_delta("4"), StagedTagKind::Update), + ]); + assert_eq!(folded, Some(FoldedTag::Dml(outcome("INSERT", 1)))); + } + + /// The task order does not matter: a balance move folded first still + /// adds nothing. + #[test] + fn a_derived_write_folded_first_adds_nothing() { + let folded = fold_staged(&[ + (balance_delta("-4"), StagedTagKind::Update), + (source_delete(), StagedTagKind::Delete), + ]); + assert_eq!(folded, Some(FoldedTag::Dml(outcome("DELETE", 1)))); + } + + /// An implicit graph edge write beside the user's own delete is derived + /// too. Its count never adds to the deleted rows. + #[test] + fn an_implicit_edge_delete_adds_no_count() { + let edge_cleanup = PhysicalPlan::Graph(GraphOp::EdgeDeleteBatch { edges: Vec::new() }); + let folded = fold_staged(&[ + (source_delete(), StagedTagKind::Delete), + (edge_cleanup, StagedTagKind::Delete), + ]); + assert_eq!(folded, Some(FoldedTag::Dml(outcome("DELETE", 1)))); + } + + /// A statement that is only a derived-shaped write is the user's own + /// write, so it counts. + #[test] + fn a_lone_derived_shaped_write_counts() { + let folded = fold_staged(&[(balance_delta("4"), StagedTagKind::Update)]); + assert_eq!(folded, Some(FoldedTag::Dml(outcome("UPDATE", 1)))); + } + #[test] fn truncate_is_the_only_count_less_verb() { assert!(!outcome("TRUNCATE", 0).carries_count()); diff --git a/nodedb/src/control/server/response_shape/types/mod.rs b/nodedb/src/control/server/response_shape/types/mod.rs index 96176689b..33c0bfc5e 100644 --- a/nodedb/src/control/server/response_shape/types/mod.rs +++ b/nodedb/src/control/server/response_shape/types/mod.rs @@ -6,7 +6,7 @@ pub mod dml_outcome; pub mod plan_kind; pub mod shaped; -pub use dml_outcome::{DmlFoldError, DmlOutcome, FoldedTag, StatementTag}; +pub use dml_outcome::{DmlFoldError, DmlOutcome, FoldedTag, StatementTag, TaskTagRole}; pub(crate) use dml_outcome::{ dml_outcome_by_op, dml_outcome_from_payload, payload_to_dml_outcome, replaced_write_outcome, staged_dml_outcome, diff --git a/nodedb/src/control/state/calvin_counters.rs b/nodedb/src/control/state/calvin_counters.rs index c959c6df6..03d0552e0 100644 --- a/nodedb/src/control/state/calvin_counters.rs +++ b/nodedb/src/control/state/calvin_counters.rs @@ -25,4 +25,8 @@ pub struct CalvinCounters { /// by dispatching a drop of their commit-pending buffer, mirroring /// [`CalvinCounters::commits_flushed`]. pub commits_dropped: Arc, + /// Count of stamped Calvin redo copies that applied on this node and + /// installed nothing: their position was applied, or another copy held + /// its claim. + pub redo_copies_skipped: Arc, } diff --git a/nodedb/src/control/state/calvin_local.rs b/nodedb/src/control/state/calvin_local.rs index e9808ec21..b7fd2525e 100644 --- a/nodedb/src/control/state/calvin_local.rs +++ b/nodedb/src/control/state/calvin_local.rs @@ -107,6 +107,7 @@ impl CalvinLocalState { read_set_validation_failures: Arc::new(AtomicU64::new(0)), commits_flushed: Arc::new(AtomicU64::new(0)), commits_dropped: Arc::new(AtomicU64::new(0)), + redo_copies_skipped: Arc::new(AtomicU64::new(0)), }, apply_results: CalvinApplySidecar::default(), lock_managers: Arc::new(Mutex::new(BTreeMap::new())), diff --git a/nodedb/src/control/wal_replication/decode/transaction_redo.rs b/nodedb/src/control/wal_replication/decode/transaction_redo.rs index 43c7a0ac5..5fb824508 100644 --- a/nodedb/src/control/wal_replication/decode/transaction_redo.rs +++ b/nodedb/src/control/wal_replication/decode/transaction_redo.rs @@ -167,7 +167,6 @@ mod tests { sum_targets: vec![RedoSumTargets { collection: "entries".into(), resolved: vec![ResolvedSumTarget::new("accounts", "a1", Surrogate::new(9))], - deferred: vec!["audit".into()], }], identities: vec![CarriedIdentity { collection: "entries".into(), diff --git a/nodedb/src/control/wal_replication/transaction_redo/sum_targets.rs b/nodedb/src/control/wal_replication/transaction_redo/sum_targets.rs index dcce2dc28..159880744 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/sum_targets.rs +++ b/nodedb/src/control/wal_replication/transaction_redo/sum_targets.rs @@ -9,9 +9,15 @@ use std::collections::BTreeMap; use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan, RedoSumTargets, ResolvedSumTarget}; -/// Every resolved and deferred sum target `plans` carry, one entry per source -/// collection, in collection order. A target resolved by several plans is -/// listed once. +/// Every sum target `plans` fold on the source's own core, one entry per +/// source collection, in collection order. A target resolved by several plans +/// is listed once. +/// +/// The install folds every row of a collection against this one merged +/// table. So a target a plan defers onto its own `ApplyBalanceDelta` task +/// never enters it. Otherwise a sibling row of the same collection, a DELETE +/// next to an INSERT on the same account, folds that target on the source's +/// core as well as the shipped task moving it. pub fn redo_sum_targets(plans: &[PhysicalPlan]) -> Vec { let mut by_collection: BTreeMap = BTreeMap::new(); for plan in plans { @@ -21,7 +27,11 @@ pub fn redo_sum_targets(plans: &[PhysicalPlan]) -> Vec { let Some((collection, resolved, deferred)) = document_sum_targets(op) else { continue; }; - if resolved.is_empty() && deferred.is_empty() { + let mut folded = resolved + .iter() + .filter(|target| !shipped_on_own_task(target, deferred)) + .peekable(); + if folded.peek().is_none() { continue; } let entry = by_collection @@ -29,22 +39,25 @@ pub fn redo_sum_targets(plans: &[PhysicalPlan]) -> Vec { .or_insert_with(|| RedoSumTargets { collection: collection.to_string(), resolved: Vec::new(), - deferred: Vec::new(), }); - for target in resolved { + for target in folded { if !entry.resolved.contains(target) { entry.resolved.push(target.clone()); } } - for target in deferred { - if !entry.deferred.contains(target) { - entry.deferred.push(target.clone()); - } - } } by_collection.into_values().collect() } +/// Whether `target` names a target collection its plan defers onto its own +/// `ApplyBalanceDelta` task. +fn shipped_on_own_task(target: &ResolvedSumTarget, deferred: &[String]) -> bool { + target + .target_collection + .as_deref() + .is_some_and(|collection| deferred.iter().any(|d| d == collection)) +} + type SumTargetSlots<'a> = (&'a str, &'a [ResolvedSumTarget], &'a [String]); /// The source collection and sum-target slots of one document write. @@ -159,6 +172,54 @@ mod tests { assert_eq!(targets[0].resolved, vec![target]); } + fn insert_deferring(target: ResolvedSumTarget, deferred: &str) -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::PointInsert { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "entries"), + document_id: "e2".into(), + value: Vec::new(), + if_absent: false, + surrogate: Surrogate::new(2), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: vec![target], + deferred_sum_targets: vec![deferred.to_string()], + }) + } + + /// An insert that ships its balance on its own task lends no resolution + /// to the merged table. A delete of the same account in the same + /// transaction ships its own balance too, and the merged table must not + /// tell the install to fold it on the source's core as well. + #[test] + fn a_target_shipped_on_its_own_task_is_not_folded_by_a_sibling_row() { + let shipped = ResolvedSumTarget::new("accounts", "a1", Surrogate::new(9)); + let plans = vec![ + insert_deferring(shipped, "accounts"), + PhysicalPlan::Document(DocumentOp::PointDelete { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "entries"), + document_id: "e1".into(), + surrogate: Some(Surrogate::new(1)), + pk_bytes: b"e1".to_vec(), + returning: None, + rls_filters: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::NoPolicyApplies, + resolved_sum_targets: Vec::new(), + }), + ]; + assert!(redo_sum_targets(&plans).is_empty()); + } + + /// A deferral names one target collection. The same insert still folds a + /// target it does not defer. + #[test] + fn a_deferral_keeps_the_targets_it_does_not_name() { + let kept = ResolvedSumTarget::new("audit_totals", "a1", Surrogate::new(7)); + let plan = insert_deferring(kept.clone(), "accounts"); + let targets = redo_sum_targets(&[plan]); + assert_eq!(targets.len(), 1); + assert_eq!(targets[0].resolved, vec![kept]); + } + #[test] fn a_write_with_no_binding_contributes_nothing() { let plan = PhysicalPlan::Document(DocumentOp::PointDelete { diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_fold_tests.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_fold_tests.rs index 476d72b94..eaca0fb7d 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_fold_tests.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_fold_tests.rs @@ -21,19 +21,38 @@ use crate::types::{Lsn, VShardId}; use crate::wal::{CalvinStamp, RedoRecord, WriteGroup, WriteGroupRecord}; const DB: u64 = 0; -pub(super) const TID: u64 = 1; +pub(in crate::data::executor) const TID: u64 = 1; /// Source and target share a vShard, so the inline fold writes the target /// on this core. -pub(super) const SOURCE: &str = "local_charges"; -pub(super) const TARGET: &str = "local_balances"; +pub(in crate::data::executor) const SOURCE: &str = "local_charges"; +pub(in crate::data::executor) const TARGET: &str = "local_balances"; const ACCOUNT: &str = "a1"; pub(super) const TARGET_SURROGATE: Surrogate = Surrogate(4242); const LSN: u64 = 70; /// A core whose source collection folds `amount` into the target's /// `balance`, with the target row seeded at 100. -pub(super) fn core_with_sum(dir: &std::path::Path) -> (CoreLoop, Box) { +pub(in crate::data::executor) fn core_with_sum( + dir: &std::path::Path, +) -> (CoreLoop, Box) { let (mut core, req, resp) = make_core_with_dir(dir); + configure_sum(&mut core); + let seed = serde_json::json!({"id": ACCOUNT, "balance": "100"}); + core.sparse + .put( + DB, + TID, + TARGET, + &nodedb_types::StorageKey::for_surrogate(TARGET_SURROGATE), + &doc_format::encode_to_msgpack(&seed), + ) + .expect("seed target row"); + (core, Box::new((req, resp))) +} + +/// Declare on `core` that the source collection folds `amount` into the +/// target's `balance`. The target row is left as the stores hold it. +pub(in crate::data::executor) fn configure_sum(core: &mut CoreLoop) { core.doc_configs.insert( (DatabaseId::DEFAULT, TenantId::new(TID), TARGET.to_string()), CollectionConfig::new(TARGET), @@ -50,29 +69,17 @@ pub(super) fn core_with_sum(dir: &std::path::Path) -> (CoreLoop, Box Vec { +pub(in crate::data::executor) fn sum_targets() -> Vec { vec![RedoSumTargets { collection: SOURCE.to_string(), resolved: vec![ResolvedSumTarget::new(TARGET, ACCOUNT, TARGET_SURROGATE)], - deferred: Vec::new(), }] } /// A Calvin redo record writing one source row of `amount` 25. -pub(super) fn calvin_redo() -> Vec { +pub(in crate::data::executor) fn calvin_redo() -> Vec { let body = doc_format::encode_to_msgpack(&serde_json::json!({ "account_id": ACCOUNT, "amount": 25, @@ -94,7 +101,7 @@ pub(super) fn calvin_redo() -> Vec { .expect("encode redo") } -pub(super) fn balance(core: &CoreLoop) -> Option { +pub(in crate::data::executor) fn balance(core: &CoreLoop) -> Option { let stored = core .sparse .get( diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/cross_shard_fold_tests.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/cross_shard_fold_tests.rs new file mode 100644 index 000000000..975112b54 --- /dev/null +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/cross_shard_fold_tests.rs @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A committed record never folds a cross-shard sum target on the source's +//! core. +//! +//! A cross-shard balance travels on an `ApplyBalanceDelta` task homed on the +//! target's vShard. An INSERT says so by its deferral list. A DELETE says so +//! by leaving its join value unresolved. The install folds every row of one +//! collection against one merged table. So an INSERT and a DELETE of one +//! account in one transaction must leave that account out of the table. +//! Otherwise the DELETE row folds the account here as well, and the shipped +//! task moves it a second time. + +use nodedb_physical::physical_plan::{ + DocumentOp, MaterializedSumBinding, PhysicalPlan, ResolvedSumTarget, +}; +use nodedb_types::{ + CollectionKey, DatabaseId, QualifiedCollection, RlsWriteCheck, StorageKey, Surrogate, TenantId, +}; + +use crate::bridge::envelope::Status; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; +use crate::data::executor::doc_format; +use crate::engine::document::store::CollectionConfig; + +const DB: u64 = 0; +const TID: u64 = 1; +/// The sum's source collection. +const SOURCE: &str = "local_charges"; +/// A target on another vShard than [`SOURCE`]. +const TARGET: &str = "remote_balances"; +const ACCOUNT: &str = "a1"; +const TARGET_SURROGATE: Surrogate = Surrogate(4242); +/// The stored row the transaction deletes. +const DELETED: Surrogate = Surrogate(1); +/// The row the transaction inserts. +const INSERTED: Surrogate = Surrogate(2); + +/// A core where [`SOURCE`] sums `amount` into [`TARGET`]'s `balance`, with +/// the target row at 10 and one source row of 10 for the account. +fn seeded_core(dir: &std::path::Path) -> (CoreLoop, Box) { + let (mut core, req, resp) = make_core_with_dir(dir); + core.doc_configs.insert( + (DatabaseId::DEFAULT, TenantId::new(TID), TARGET.to_string()), + CollectionConfig::new(TARGET), + ); + let mut source = CollectionConfig::new(SOURCE); + source.enforcement.materialized_sum_sources = vec![MaterializedSumBinding { + target_collection: TARGET.to_string(), + target_column: "balance".to_string(), + join_column: "account_id".to_string(), + value_expr: nodedb_query::expr::SqlExpr::Column("amount".to_string()), + declared_primary_key: None, + }]; + core.doc_configs.insert( + (DatabaseId::DEFAULT, TenantId::new(TID), SOURCE.to_string()), + source, + ); + // The target row lives in this core's store, so a fold on this core is + // visible as a moved balance rather than a refusal. + put_row( + &mut core, + TARGET, + TARGET_SURROGATE, + &serde_json::json!({"id": ACCOUNT, "balance": "10"}), + ); + put_row(&mut core, SOURCE, DELETED, &entry(10)); + (core, Box::new((req, resp))) +} + +fn entry(amount: i64) -> serde_json::Value { + serde_json::json!({"account_id": ACCOUNT, "amount": amount}) +} + +fn put_row(core: &mut CoreLoop, collection: &str, surrogate: Surrogate, doc: &serde_json::Value) { + core.sparse + .put( + DB, + TID, + collection, + &StorageKey::for_surrogate(surrogate), + &doc_format::encode_to_msgpack(doc), + ) + .expect("seed row"); +} + +fn row(core: &CoreLoop, collection: &str, surrogate: Surrogate) -> Option { + let stored = core + .sparse + .get(DB, TID, collection, &StorageKey::for_surrogate(surrogate)) + .expect("read row")?; + doc_format::decode_document(&stored).ok() +} + +/// The source slice of `BEGIN; INSERT ...; DELETE ...; COMMIT` as the +/// Control Plane plans it. The INSERT keeps its resolution and defers the +/// target. The DELETE's balance was settled from its stored row, so its join +/// value is gone from its resolution. +fn source_slice() -> Vec { + let collection = QualifiedCollection::new(DatabaseId::DEFAULT, SOURCE); + vec![ + PhysicalPlan::Document(DocumentOp::PointInsert { + collection: collection.clone(), + document_id: "e1".to_string(), + value: doc_format::encode_to_msgpack(&entry(4)), + if_absent: false, + surrogate: INSERTED, + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: vec![ResolvedSumTarget::new(TARGET, ACCOUNT, TARGET_SURROGATE)], + deferred_sum_targets: vec![TARGET.to_string()], + }), + PhysicalPlan::Document(DocumentOp::PointDelete { + collection, + document_id: "e0".to_string(), + surrogate: Some(DELETED), + pk_bytes: Vec::new(), + returning: None, + rls_filters: Vec::new(), + rls_write_check: RlsWriteCheck::NoPolicyApplies, + resolved_sum_targets: Vec::new(), + }), + ] +} + +/// The premise: the target is cross-shard, so its balance moves only on its +/// own task. +#[test] +fn the_fixture_target_is_cross_shard() { + assert!( + !crate::query::sum_target_is_co_resident( + CollectionKey::from_bare(DatabaseId::DEFAULT, SOURCE), + TARGET, + ), + "'{TARGET}' must not share '{SOURCE}''s vShard" + ); +} + +/// The source slice of an INSERT and a DELETE on one account installs both +/// rows and leaves the cross-shard balance to the shipped tasks. +#[test] +fn a_delete_next_to_an_insert_of_its_account_folds_no_cross_shard_balance() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _ends) = seeded_core(dir.path()); + + let response = core.commit_plans_for_test(&make_default_task(), TID, &source_slice(), 30); + + assert_eq!(response.status, Status::Ok, "{:?}", response.error_code); + assert!(row(&core, SOURCE, DELETED).is_none(), "e0 is deleted"); + assert!(row(&core, SOURCE, INSERTED).is_some(), "e1 is inserted"); + assert_eq!( + row(&core, TARGET, TARGET_SURROGATE) + .as_ref() + .and_then(|doc| doc.get("balance")) + .and_then(|balance| balance.as_str()), + Some("10"), + "the source install moves no cross-shard balance; the shipped tasks move it once" + ); +} diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/document.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/document.rs index 71b99df6f..861a91ac5 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/document.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/document.rs @@ -92,7 +92,7 @@ impl CoreLoop { row: &CommittedDocWrite<'_>, value: &[u8], ) -> crate::Result<()> { - let (resolved, deferred) = self.committed_sum_targets(row.collection); + let resolved = self.committed_sum_targets(row.collection); let surrogate = Surrogate::new(row.surrogate); let storage_key = StorageKey::for_surrogate(surrogate); let wal_lsn = (row.record_lsn != 0).then(|| Lsn::new(row.record_lsn)); @@ -101,7 +101,8 @@ impl CoreLoop { tid: row.tenant_id, collection: row.collection, resolved_targets: &resolved, - deferred_sum_targets: &deferred, + // A committed record defers a target by omission from `resolved`. + deferred_sum_targets: &[], wal_lsn, }; @@ -238,7 +239,7 @@ impl CoreLoop { } fn committed_document_delete(&mut self, row: &CommittedDocWrite<'_>) -> crate::Result { - let (resolved, _) = self.committed_sum_targets(row.collection); + let resolved = self.committed_sum_targets(row.collection); let surrogate = Surrogate::new(row.surrogate); let storage_key = StorageKey::for_surrogate(surrogate); let hook_ctx = HookCtx { @@ -246,7 +247,7 @@ impl CoreLoop { tid: row.tenant_id, collection: row.collection, resolved_targets: &resolved, - // A delete is deferred by omission from `resolved`, never by list. + // A committed record defers a target by omission from `resolved`. deferred_sum_targets: &[], wal_lsn: (row.record_lsn != 0).then(|| Lsn::new(row.record_lsn)), }; @@ -338,10 +339,7 @@ impl CoreLoop { fn committed_sum_targets( &self, collection: &str, - ) -> ( - Vec, - Vec, - ) { + ) -> Vec { self.redo_apply .scope .as_ref() diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/mod.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/mod.rs index 042b0b49f..8c5accbb2 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/mod.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/mod.rs @@ -19,13 +19,17 @@ //! and restart replay applies them. //! - [`calvin_install_tests`]: a Calvin slice installs from its stamped //! redo entry. +//! - [`cross_shard_fold_tests`]: a record never folds a cross-shard sum +//! target on the source's core. //! - [`unique_handover_tests`]: UNIQUE judged on a record's post-state. #[cfg(test)] -mod calvin_fold_tests; +pub(in crate::data::executor) mod calvin_fold_tests; mod calvin_install; #[cfg(test)] mod calvin_install_tests; +#[cfg(test)] +mod cross_shard_fold_tests; mod document; mod entry; mod events; diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs index a6f7fb22c..df380a241 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs @@ -186,15 +186,16 @@ impl RedoApplyScope { } } - /// The resolved and deferred sum targets for writes to `collection`. + /// The sum targets writes to `collection` fold on this core. A + /// cross-shard target with no entry travels on its own task. pub(in crate::data::executor) fn sum_targets_for( &self, collection: &str, - ) -> (Vec, Vec) { - match self.sum_targets.get(collection) { - Some(targets) => (targets.resolved.clone(), targets.deferred.clone()), - None => (Vec::new(), Vec::new()), - } + ) -> Vec { + self.sum_targets + .get(collection) + .map(|targets| targets.resolved.clone()) + .unwrap_or_default() } /// Keep the first error; a later one is a consequence of it. diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/validate.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/validate.rs index 56be3e760..c4489648e 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/validate.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/validate.rs @@ -68,7 +68,7 @@ impl CoreLoop { continue; }; let bitemporal = self.is_bitemporal(database_id, tid, collection); - let (resolved, _) = apply_scope.sum_targets_for(collection); + let resolved = apply_scope.sum_targets_for(collection); let scope = CollectionScope { database_id, tid, diff --git a/nodedb/src/data/executor/write_set_crash_tests.rs b/nodedb/src/data/executor/write_set_crash_tests.rs index a507478e2..d3615b2d3 100644 --- a/nodedb/src/data/executor/write_set_crash_tests.rs +++ b/nodedb/src/data/executor/write_set_crash_tests.rs @@ -9,11 +9,13 @@ //! onto an empty base (a point-in-time restore), and the settled WAL the //! event stream is rebuilt from. -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::path::Path; use std::time::{Duration, Instant}; -use nodedb_physical::physical_plan::{DocumentOp, UpdateValue}; +use nodedb_physical::physical_plan::{ + CalvinInstall, CalvinReplySpec, DocumentOp, MetaOp, RedoOrigin, UpdateValue, +}; use nodedb_types::{QualifiedCollection, RlsWriteCheck, Surrogate, Value}; use nodedb_wal::{TombstoneSet, WalRecord}; @@ -23,11 +25,16 @@ use crate::bridge::dispatch::JournalGroup; use crate::bridge::envelope::{ Admission, PhysicalPlan, Priority, Request, Response, Status, WriteSetEntry, }; +use crate::control::cluster::calvin::scheduler::recover_all_applied; +use crate::control::security::catalog::SystemCatalog; use crate::control::server::wal_dispatch::{ GroupOrigin, WalAppendRequest, WriteSetTarget, append_group_origin, append_group_parts, }; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::tests::make_core_with_dir; +use crate::data::executor::handlers::transaction::redo_apply::calvin_fold_tests::{ + SOURCE, balance, calvin_redo, configure_sum, core_with_sum, sum_targets, +}; use crate::data::executor::task::ExecutionTask; use crate::types::{DatabaseId, ReadConsistency, RequestId, TenantId, TraceId, VShardId}; use crate::wal::GroupMembership; @@ -49,10 +56,15 @@ struct Live { impl Live { fn open() -> Self { + Self::open_with(|dir| make_core_with_dir(dir).0) + } + + /// A live node whose core `make` builds over its store directory. + fn open_with(make: impl FnOnce(&Path) -> CoreLoop) -> Self { let wal_dir = tempfile::tempdir().expect("tempdir"); let wal = WalManager::open_for_testing(&wal_dir.path().join("wal")).expect("open wal"); let store_dir = tempfile::tempdir().expect("tempdir"); - let (core, _req, _resp) = make_core_with_dir(store_dir.path()); + let core = make(store_dir.path()); Self { wal, wal_dir, @@ -106,6 +118,12 @@ impl Live { /// `stored`, the core stores its write set. The parts are never /// appended: the crash comes first. fn journalled_until_crash(&mut self, plan: PhysicalPlan, stored: bool) -> GroupOrigin { + self.journalled_in(ORDERS, plan, stored) + } + + /// [`Self::journalled_until_crash`] for a write whose group falls back + /// to `collection`. + fn journalled_in(&mut self, collection: &str, plan: PhysicalPlan, stored: bool) -> GroupOrigin { let origin = self.append_origin(&plan); let task = ExecutionTask::new(request(plan, origin)); self.core.begin_journalled(); @@ -114,7 +132,7 @@ impl Live { if stored { let group = JournalGroup { origin: origin.lsn, - collection: ORDERS.to_string(), + collection: collection.to_string(), apply_key: NO_APPLY_KEY, commit_hlc: None, change_position: None, @@ -442,3 +460,81 @@ fn a_settled_group_needs_nothing_at_boot() { "a whole WAL needs no record" ); } + +/// The stamped install of the Calvin slice at `(3, 0)` on vShard 0. Its +/// record writes one source row, and its sum target folds 25 into the +/// target's balance. +fn calvin_install() -> PhysicalPlan { + PhysicalPlan::Meta(MetaOp::ApplyTransactionRedo { + redo: calvin_redo(), + collections: vec![SOURCE.to_string()], + sum_targets: sum_targets(), + origin: RedoOrigin::Commit, + calvin: Some(CalvinInstall { + epoch: 3, + position: 0, + epoch_system_ms: 0, + reply: CalvinReplySpec::Count(Vec::new()), + user_write: true, + }), + }) +} + +/// The target's balance after replaying `records` onto the stores at `dir`, +/// on a core that declares the sum. +fn balance_after_replay(dir: &Path, records: &[WalRecord]) -> Option { + let (mut core, _req, _resp) = make_core_with_dir(dir); + configure_sum(&mut core); + core.replay_all_wal(records, 1, &TombstoneSet::new()) + .expect("replay"); + balance(&core) +} + +/// The Calvin positions boot recovery finds applied on vShard 0 in `wal`. +fn recovered_positions(wal: &WalManager) -> BTreeSet<(u64, u32)> { + let catalog = SystemCatalog::open_in_memory().expect("in-memory catalog"); + recover_all_applied(wal, &catalog, &|_| None) + .expect("recover applied positions") + .remove(&0) + .map(|recovered| recovered.applied_tail) + .unwrap_or_default() +} + +/// A crash cuts the origin of a stamped Calvin install from the WAL after +/// the install folded its sum and stored its write set. Boot journals the +/// stamped record again, then the fold rows as its parts. Recovery finds +/// the position applied, and replay keeps the single-apply total. +#[test] +fn a_crash_that_cuts_a_calvin_redo_origin_journals_it_again() { + let mut live = Live::open_with(|dir| core_with_sum(dir).0); + let cut = live.wal_image(); + live.journalled_in(SOURCE, calvin_install(), true); + assert_eq!(balance(&live.core).as_deref(), Some("125"), "the live fold"); + let stores = live.crash_image(); + + let cut_wal = WalManager::open_for_testing(&cut.path().join("wal")).expect("open cut wal"); + assert!( + recovered_positions(&cut_wal).is_empty(), + "the crash cuts the stamped record" + ); + let records = settle(&cut_wal, stores.path()); + assert!( + recovered_positions(&cut_wal).contains(&(3, 0)), + "recovery finds the stamp of the record boot journals" + ); + assert!( + membership(&records).broken_through(u64::MAX).is_empty(), + "the journalled record's group is whole" + ); + assert_eq!( + balance_after_replay(stores.path(), &records).as_deref(), + Some("125"), + "restart replay onto the cut stores keeps the single-apply total" + ); + let base = tempfile::tempdir().expect("tempdir"); + assert_eq!( + balance_after_replay(base.path(), &records).as_deref(), + Some("125"), + "a point-in-time restore reaches the single-apply total" + ); +} diff --git a/nodedb/tests/crash_calvin_redo_cut.rs b/nodedb/tests/crash_calvin_redo_cut.rs new file mode 100644 index 000000000..1384dfe47 --- /dev/null +++ b/nodedb/tests/crash_calvin_redo_cut.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A Calvin install cut by a crash after its write set is stored applies +//! once. +//! +//! The source collection carries two materialized sums. One target shares +//! the source's vShard, so the source slice's install folds it inline. The +//! other target homes on another vShard, so its balance travels as an +//! `ApplyBalanceDelta` slice. One source `INSERT` therefore commits through +//! the Calvin scheduler, and each slice installs from a stamped redo entry. +//! +//! The process aborts at `core::after_capture::`: the core stored the +//! source install's write set, and its parts are not journalled or synced. +//! Boot journals the install from the stored write set. The stamp of that +//! record marks the position applied, so a redelivered copy of the entry +//! installs nothing, and both balances hold the single-apply total. +//! +//! The checkpoint interval is pushed beyond the test's runtime, so the values +//! read after the crash come from WAL replay alone. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +mod crash_harness; + +use std::collections::BTreeSet; +use std::time::{Duration, Instant}; + +use crash_harness::log_fields::same_value; +use crash_harness::vshards::names_on_distinct_vshards; +use crash_harness::{CrashHarness, diagnostics}; +use nodedb::control::cluster::calvin::scheduler::{CalvinAppliedLedgers, recover_all_applied}; +use nodedb::control::security::catalog::SystemCatalog; +use nodedb::wal::WalManager; +use nodedb_types::id::DatabaseId; + +/// A checkpoint between the crash and the reads holds the balances +/// independent of the WAL, and a doubled fold then hides in it. +const NO_CHECKPOINT_SECS: &str = "3600"; + +/// How long the Calvin sequencer may take to elect its leader after a boot. +const CALVIN_READY_TIMEOUT: Duration = Duration::from_secs(30); + +/// How long the process may take to abort once the insert is sent. +const CRASH_TIMEOUT: Duration = Duration::from_secs(60); + +/// How long a balance may take to move off its seed after the restart. +const SETTLE_TIMEOUT: Duration = Duration::from_secs(30); + +const ACCOUNT: &str = "acc-1"; +const SEED: &str = "100"; +const AMOUNT: &str = "25"; +const SINGLE_APPLY: &str = "125"; + +fn vshard_of(name: &str) -> u32 { + nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, name) + .vshard() + .as_u32() +} + +/// The first name `_` that homes on `vshard`. +fn name_on_vshard(prefix: &str, vshard: u32) -> String { + (0..1u32 << 16) + .map(|i| format!("{prefix}_{i}")) + .find(|name| vshard_of(name) == vshard) + .unwrap_or_else(|| panic!("no {prefix} name on vShard {vshard} in 65536 tries")) +} + +/// The balance of the account in `target`, once it moved off the seed or +/// `SETTLE_TIMEOUT` passed. +async fn settled_balance(h: &CrashHarness, target: &str) -> String { + let sql = format!("SELECT balance FROM {target} WHERE id = '{ACCOUNT}'"); + let deadline = Instant::now() + SETTLE_TIMEOUT; + loop { + let read = h.query_col_idx(&sql, 0).await; + assert_eq!(read.len(), 1, "{target} holds one account row: {read:?}"); + if !same_value(&read[0], SEED) || Instant::now() >= deadline { + return read[0].clone(); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +/// The applied ledgers boot fills from the stopped server's WAL and the +/// state its catalog saved. +fn boot_ledgers(h: &CrashHarness) -> CalvinAppliedLedgers { + let wal = WalManager::open_for_testing(&h.data_dir().join("wal")).expect("open the WAL"); + let catalog = SystemCatalog::open(&h.data_dir().join("system.redb")).expect("open the catalog"); + let ledgers = CalvinAppliedLedgers::default(); + let recovered = + recover_all_applied(&wal, &catalog, &|_| None).expect("recover applied positions"); + for (vshard_id, state) in recovered { + ledgers.install(vshard_id, state.fully_applied_epoch, state.applied_tail); + } + ledgers +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_calvin_install_cut_after_its_write_set_is_stored_applies_once() { + let [source, remote] = names_on_distinct_vshards(["cut_entries", "cut_remote"]); + let local = name_on_vshard("cut_local", vshard_of(&source)); + assert_ne!(vshard_of(&source), vshard_of(&remote)); + + let mut h = CrashHarness::new().with_env("NODEDB_CHECKPOINT_INTERVAL_SECS", NO_CHECKPOINT_SECS); + h.spawn(); + h.wait_ready(); + h.wait_for_calvin_ready(CALVIN_READY_TIMEOUT).await; + for target in [&local, &remote] { + h.exec(&format!( + "CREATE COLLECTION {target} (id TEXT PRIMARY KEY, owner TEXT) \ + WITH (engine='document_strict')" + )) + .await; + } + h.exec(&format!( + "CREATE COLLECTION {source} (id TEXT PRIMARY KEY, account_id TEXT, amount TEXT) \ + WITH (engine='document_strict')" + )) + .await; + for target in [&local, &remote] { + h.exec(&format!( + "ALTER COLLECTION {target} ADD COLUMN balance TEXT \ + MATERIALIZED_SUM SOURCE {source} \ + ON {source}.account_id = {target}.id VALUE {source}.amount" + )) + .await; + h.exec(&format!( + "INSERT INTO {target} (id, owner, balance) VALUES ('{ACCOUNT}', 'alice', '{SEED}')" + )) + .await; + } + + // The next boot aborts once a write to the source stored its write set. + // The abort matches only a journalled write, so boot itself passes it. + h.kill_9(); + h.set_env( + "NODEDB_FAILPOINTS", + &format!("core::after_capture::{source}=abort"), + ); + h.reopen(); + h.wait_for_calvin_ready(CALVIN_READY_TIMEOUT).await; + + // The insert's client loses its connection with the process, so its + // result says nothing. + let conn_str = h.pgwire_conn_str(); + let insert = format!( + "INSERT INTO {source} (id, account_id, amount) VALUES ('e1', '{ACCOUNT}', '{AMOUNT}')" + ); + let insert_task = tokio::spawn(async move { + let (client, connection) = tokio_postgres::connect(&conn_str, tokio_postgres::NoTls) + .await + .map_err(|e| e.to_string())?; + tokio::spawn(connection); + client + .simple_query(&insert) + .await + .map(|_| ()) + .map_err(|e| e.to_string()) + }); + h.await_self_crash(CRASH_TIMEOUT); + let marker = format!("fail_point aborting process: core::after_capture::{source}"); + assert!( + h.server_log().contains(&marker), + "the process exited, but not after the source install stored its write set.{}\n{}", + h.keep_data_dir_note(), + diagnostics::log_tail_section(&h.server_log()) + ); + let _ = insert_task.await; + + h.clear_env("NODEDB_FAILPOINTS"); + h.reopen(); + h.wait_for_calvin_ready(CALVIN_READY_TIMEOUT).await; + + for target in [&local, &remote] { + let balance = settled_balance(&h, target).await; + assert!( + same_value(&balance, SINGLE_APPLY), + "{target} holds balance {balance}, not the single-apply total {SINGLE_APPLY}.{}\n{}", + h.keep_data_dir_note(), + diagnostics::log_tail_section(&h.server_log()) + ); + } + assert_eq!( + h.query_col_idx(&format!("SELECT id FROM {source}"), 0) + .await, + vec!["e1".to_string()], + "the source row is stored once" + ); + + // The ledgers a boot fills from this data directory report the + // transaction's position applied on both of its vShards. + h.kill_9(); + let ledgers = boot_ledgers(&h); + let tail = |vshard: u32| -> BTreeSet<(u64, u32)> { + ledgers + .get(vshard) + .map(|ledger| ledger.snapshot().1) + .unwrap_or_default() + }; + let shared: Vec<(u64, u32)> = tail(vshard_of(&source)) + .intersection(&tail(vshard_of(&remote))) + .copied() + .collect(); + assert_eq!( + shared.len(), + 1, + "the cross-shard transaction is the one position applied on both vShards: {shared:?}" + ); + let (epoch, position) = shared[0]; + for vshard in [vshard_of(&source), vshard_of(&remote)] { + assert!( + ledgers + .get(vshard) + .is_some_and(|ledger| ledger.is_applied(epoch, position)), + "the ledger of vShard {vshard} reports ({epoch}, {position}) applied" + ); + } + + // One more boot replays the WAL and takes every redelivered entry again. + // A second install of either slice folds a second time, and the reads + // below then show the doubled total. + h.reopen(); + h.wait_for_calvin_ready(CALVIN_READY_TIMEOUT).await; + for target in [&local, &remote] { + let read = h + .query_col_idx( + &format!("SELECT balance FROM {target} WHERE id = '{ACCOUNT}'"), + 0, + ) + .await; + assert!( + read.len() == 1 && same_value(&read[0], SINGLE_APPLY), + "after a second restart {target} holds {read:?}, not the single-apply total \ + {SINGLE_APPLY}: a copy of the entry installed again.{}\n{}", + h.keep_data_dir_note(), + diagnostics::log_tail_section(&h.server_log()) + ); + } + assert_eq!( + h.query_col_idx(&format!("SELECT id FROM {source}"), 0) + .await, + vec!["e1".to_string()], + "after a second restart the source row is stored once" + ); +} diff --git a/nodedb/tests/wire/cases/materialized_sum_txn_command_tag.rs b/nodedb/tests/wire/cases/materialized_sum_txn_command_tag.rs new file mode 100644 index 000000000..7a9deb404 --- /dev/null +++ b/nodedb/tests/wire/cases/materialized_sum_txn_command_tag.rs @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A write to a materialized-sum source in a transaction block answers with +//! the command tag of its own rows only. +//! +//! A sum-source write whose target lives on another vShard plans a second +//! task: the balance move on the target row. That task is a derived write. +//! Its verb and its count never reach the statement's tag. An `INSERT` +//! answers `INSERT 0 1`, never `INSERT 0 2`. A `DELETE` answers `DELETE 1`, +//! never an error for mixing `DELETE` with the balance move's `UPDATE`. + +use nodedb_types::id::DatabaseId; +use tokio_postgres::SimpleQueryMessage; + +use crate::harness::TestServer; + +/// A target and source that share one vShard: the balance rides the source +/// write's own transaction. +const CO_RESIDENT: (&str, &str) = ("uo_accounts", "uo_entries"); + +/// A target and source on two vShards: the balance travels on its own task. +const CROSS_SHARD: (&str, &str) = ("ptv_acct", "ptv_post"); + +fn vshard(name: &str) -> u32 { + nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, name) + .vshard() + .as_u32() +} + +/// The premise both paths rest on: the names hash as the constants say. +#[test] +fn the_pairs_hash_to_the_paths_they_name() { + assert_eq!(vshard(CO_RESIDENT.0), vshard(CO_RESIDENT.1)); + assert_ne!( + vshard(CROSS_SHARD.0), + vshard(CROSS_SHARD.1), + "the cross-shard pair must plan a separate balance task" + ); +} + +/// Every `CommandComplete` count in `sql`'s simple-query response, in wire +/// order. +async fn command_counts(server: &TestServer, sql: &str) -> Vec { + let messages = server + .client + .simple_query(sql) + .await + .unwrap_or_else(|e| panic!("run {sql}: {e:?}")); + messages + .into_iter() + .filter_map(|m| match m { + SimpleQueryMessage::CommandComplete(n) => Some(n), + _ => None, + }) + .collect() +} + +/// Create `target` with a `balance` summed from `source.amount`, and seed +/// one account and one entry. +async fn sum_pair(server: &TestServer, target: &str, source: &str) { + server + .exec(&format!( + "CREATE COLLECTION {target} (id TEXT PRIMARY KEY, owner TEXT) \ + WITH (engine='document_strict')" + )) + .await + .unwrap(); + server + .exec(&format!( + "CREATE COLLECTION {source} (id TEXT PRIMARY KEY, account_id TEXT, amount TEXT) \ + WITH (engine='document_strict')" + )) + .await + .unwrap(); + server + .exec(&format!( + "ALTER COLLECTION {target} ADD COLUMN balance TEXT \ + MATERIALIZED_SUM SOURCE {source} \ + ON {source}.account_id = {target}.id VALUE {source}.amount" + )) + .await + .unwrap(); + server + .exec(&format!( + "INSERT INTO {target} (id, owner, balance) VALUES ('acc', 'alice', '0')" + )) + .await + .unwrap(); + server + .exec(&format!( + "INSERT INTO {source} (id, account_id, amount) VALUES ('e0', 'acc', '10')" + )) + .await + .unwrap(); +} + +/// `BEGIN; INSERT; DELETE; COMMIT` on a sum source tags each write with its +/// own row count, whether the target shares the source's vShard or not. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_sum_source_write_in_a_block_tags_only_its_own_rows() { + let server = TestServer::start().await; + for (target, source) in [CO_RESIDENT, CROSS_SHARD] { + sum_pair(&server, target, source).await; + + let sql = format!( + "BEGIN; \ + INSERT INTO {source} (id, account_id, amount) VALUES ('e1', 'acc', '4'); \ + DELETE FROM {source} WHERE id = 'e0'; \ + COMMIT" + ); + assert_eq!( + command_counts(&server, &sql).await, + vec![0, 1, 1, 0], + "BEGIN, INSERT 0 1, DELETE 1, COMMIT for {source}" + ); + + let balance = server + .query_text(&format!("SELECT balance FROM {target} WHERE id = 'acc'")) + .await + .unwrap(); + assert_eq!( + balance, + vec!["4".to_string()], + "{target} holds the inserted entry and not the deleted one" + ); + } +} diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index 4c9118e37..eb9e275cf 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -127,6 +127,7 @@ mod kv_sql_select; mod kv_transfer_decimal; mod kv_write_row_level_security; mod kv_write_row_level_security_atomics; +mod materialized_sum_txn_command_tag; mod merge_insert_atomicity; mod merge_insert_cross_engine_visibility; mod merge_insert_renamed_source_column; From 0917424b10c18794408d3523069f09a1e0bd6f0c Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sat, 10 Oct 2026 09:53:34 +0800 Subject: [PATCH 08/19] fix(planner): settle materialized sums for cross-shard expander writes Route INSERT..SELECT, MERGE and UPDATE..FROM join expanders through materialized-sum resolution and admission so sums stay correct when the expanded writes span shards. Add wire coverage for the cross-shard expander cases. --- .../control/insert_select/expand_staged.rs | 38 +-- .../src/control/insert_select/orchestrator.rs | 9 + .../merge_orchestrator/orchestrator.rs | 9 + .../control/planner/materialized_sum/mod.rs | 3 +- .../planner/materialized_sum/resolve.rs | 205 +++++++++++++- .../planner/materialized_sum/settle.rs | 14 +- .../server/native/dispatch/direct_ops.rs | 5 +- .../native/dispatch/direct_orchestrated.rs | 66 +++-- .../server/native/dispatch/sql_planned.rs | 5 +- .../server/pgwire/handler/routing/execute.rs | 6 +- .../control/server/shared/plan_admission.rs | 20 ++ .../server/shared/session/expander_stage.rs | 46 ++- .../control/server/shared/txn_route/joined.rs | 73 ++++- .../control/server/shared/txn_route/route.rs | 16 +- .../shared/write_admission/admission_keys.rs | 54 ++++ .../expand_staged_update_from_join.rs | 26 +- .../orchestrator.rs | 9 + .../enforcement/materialized_sum/apply.rs | 17 +- .../materialized_sum_cross_shard_expanders.rs | 262 ++++++++++++++++++ nodedb/tests/wire/cases/mod.rs | 1 + 20 files changed, 762 insertions(+), 122 deletions(-) create mode 100644 nodedb/tests/wire/cases/materialized_sum_cross_shard_expanders.rs diff --git a/nodedb/src/control/insert_select/expand_staged.rs b/nodedb/src/control/insert_select/expand_staged.rs index 3d1c33613..0f3a10998 100644 --- a/nodedb/src/control/insert_select/expand_staged.rs +++ b/nodedb/src/control/insert_select/expand_staged.rs @@ -30,8 +30,9 @@ use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; /// /// `task.txn_id` must be set to the active transaction so the scan folds /// earlier-staged rows. Emitted ops carry the same `txn_id` and a recomputed -/// target vShard (as the MERGE / `UPDATE ... FROM` expanders do). The caller -/// stages + buffers each returned op. +/// target vShard (as the MERGE / `UPDATE ... FROM` expanders do). The ops carry +/// no sum or period-lock resolution: the caller runs the statement pass on +/// them, then stages + buffers each one. pub(crate) async fn resolve_and_emit_insert_select_ops( state: &SharedState, tenant_id: TenantId, @@ -76,35 +77,6 @@ pub(crate) async fn resolve_and_emit_insert_select_ops( )? .vshard(); - // Resolve materialized-sum targets: these ops stage directly, bypassing - // statement-level resolution, so without this a bound target collection - // folds against an empty resolution. - let sum_bodies: Vec<&[u8]> = rows.iter().map(|(_, value, _)| value.as_slice()).collect(); - let mut resolved_sum_targets = - crate::control::planner::materialized_sum::resolve_sum_targets_for_bodies( - state, - &sum_bodies, - target_collection.as_str(), - tenant_id, - task.database_id, - crate::types::TraceId::ZERO, - ) - .await?; - // Period-lock target for the same rows, in the SAME slot — a target - // collection under a period lock reads its reference row's surrogate off - // `resolved_sum_targets` exactly like a materialized-sum fold does. - resolved_sum_targets.extend( - crate::control::planner::period_lock::resolve_period_lock_targets_for_bodies( - state, - &sum_bodies, - target_collection.as_str(), - tenant_id, - task.database_id, - crate::types::TraceId::ZERO, - ) - .await?, - ); - let mut out: Vec = Vec::with_capacity(rows.len()); for (document_id, value, surrogate) in rows { out.push(PhysicalTask { @@ -121,7 +93,9 @@ pub(crate) async fn resolve_and_emit_insert_select_ops( // orchestrator's paged batch insert. returning: None, rls_filters: Vec::new(), - resolved_sum_targets: resolved_sum_targets.clone(), + // Filled by the caller's sum and period-lock pass, the one + // a plain INSERT takes. + resolved_sum_targets: Vec::new(), deferred_sum_targets: Vec::new(), }), post_set_op: PostSetOp::None, diff --git a/nodedb/src/control/insert_select/orchestrator.rs b/nodedb/src/control/insert_select/orchestrator.rs index a44dd75fa..e59887a91 100644 --- a/nodedb/src/control/insert_select/orchestrator.rs +++ b/nodedb/src/control/insert_select/orchestrator.rs @@ -84,6 +84,15 @@ pub(crate) async fn run_insert_select( database_id: DatabaseId, req: &CopyRequest<'_>, ) -> crate::Result { + // Each page applies on the target's vShard alone, so it folds only + // co-resident balances. + crate::control::planner::materialized_sum::refuse_cross_shard_orchestration( + state, + req.target_collection, + tenant_id, + database_id, + "INSERT ... SELECT", + )?; let spec = resolve_copy_spec( state, tenant_id, diff --git a/nodedb/src/control/merge_orchestrator/orchestrator.rs b/nodedb/src/control/merge_orchestrator/orchestrator.rs index 4ff1ed673..41149f8d7 100644 --- a/nodedb/src/control/merge_orchestrator/orchestrator.rs +++ b/nodedb/src/control/merge_orchestrator/orchestrator.rs @@ -121,6 +121,15 @@ pub async fn run_authorized_merge( /// Returns the `{"affected": N}` (or RETURNING-rows) response the Data-Plane /// merge handler produces, so the dispatch loops render the same command tag. pub(crate) async fn run_merge(state: &SharedState, args: MergeArgs<'_>) -> crate::Result { + // The apply lands on the target's vShard alone, so it folds only + // co-resident balances. + crate::control::planner::materialized_sum::refuse_cross_shard_orchestration( + state, + args.target_collection, + args.tenant_id, + args.database_id, + "MERGE", + )?; let catalog = state.credentials.catalog(); let target_bare = bare_collection_name(args.database_id, args.target_collection); let target = catalog diff --git a/nodedb/src/control/planner/materialized_sum/mod.rs b/nodedb/src/control/planner/materialized_sum/mod.rs index c4ea272cd..67b68e5f3 100644 --- a/nodedb/src/control/planner/materialized_sum/mod.rs +++ b/nodedb/src/control/planner/materialized_sum/mod.rs @@ -29,6 +29,7 @@ pub use cross_shard::append_cross_shard_balance_tasks; pub use extract::join_value_from_body; pub use index::MaterializedSumIndex; pub use resolve::{ - resolve_materialized_sum_targets, resolve_sum_targets_for_bodies, source_drives_bindings, + drives_cross_shard_sum, refuse_cross_shard_orchestration, resolve_materialized_sum_targets, + resolve_sum_targets_for_bodies, source_drives_bindings, }; pub use resolve_target::ResolvedTargets; diff --git a/nodedb/src/control/planner/materialized_sum/resolve.rs b/nodedb/src/control/planner/materialized_sum/resolve.rs index 81eb0f526..5c48c7a61 100644 --- a/nodedb/src/control/planner/materialized_sum/resolve.rs +++ b/nodedb/src/control/planner/materialized_sum/resolve.rs @@ -280,12 +280,14 @@ fn set_resolved(op: &mut DocumentOp, resolved: Vec) { /// Resolve the materialized-sum targets for a set of row BODIES of one source /// collection, for a caller that holds the bodies itself rather than a plan. /// -/// This is the seam the Control-Plane orchestrators use. `INSERT ... SELECT`, -/// `MERGE` and `UPDATE ... FROM` all resolve their rows on the Control Plane and -/// re-issue concrete work (a `BatchInsert` page, an APPLY pass, a write pass) -/// through `dispatch_local`, which never passes through +/// This is the seam the autocommit Control-Plane orchestrators use. +/// `INSERT ... SELECT`, `MERGE` and `UPDATE ... FROM` resolve their rows on the +/// Control Plane and re-issue concrete work (a `BatchInsert` page, an APPLY +/// pass, a write pass) that never passes through /// [`resolve_materialized_sum_targets`]. Without this they will ship an empty -/// resolution and the Data-Plane fold will have no target to address. +/// resolution and the Data-Plane fold will have no target to address. Every +/// binding they resolve is co-resident: +/// [`refuse_cross_shard_orchestration`] refuses the others first. /// /// `source_collection` is the db-qualified name as it appears on the plan. /// Returns an empty vec — and issues no lookup at all — when the collection @@ -341,6 +343,59 @@ pub fn source_drives_bindings( ) } +/// Whether `source_collection` drives a materialized sum whose target does not +/// share its vShard. `source_collection` is the db-qualified plan name. +/// +/// Such a balance cannot ride a write applied on the source's vShard alone. +/// It ships on an `ApplyBalanceDelta` task homed on the target, so the write +/// that moves it must run where Calvin can commit the pair together. +pub fn drives_cross_shard_sum( + state: &SharedState, + source_collection: &str, + tenant_id: TenantId, + database_id: DatabaseId, +) -> crate::Result { + let Some(bindings) = source_drives_bindings(state, source_collection, tenant_id, database_id)? + else { + return Ok(false); + }; + let source = nodedb_types::CollectionKey::from_qualified_str(database_id, source_collection)?; + Ok(bindings.iter().any(|binding| { + !crate::query::sum_target_is_co_resident(source, &binding.target_collection) + })) +} + +/// Refuse a Control-Plane orchestrated write into `source_collection` when it +/// drives a cross-shard materialized sum. `statement` names the statement +/// kind for the error. +/// +/// An orchestrator applies on the written collection's vShard alone. A +/// cross-shard balance travels on an `ApplyBalanceDelta` task that only a +/// transaction commits with the source rows. So the SQL and native protocols +/// run such a statement in an implicit transaction and never reach the +/// orchestrator with it. A route that reaches it anyway is refused here: +/// folding the balance on the source's core writes a row that vShard does +/// not own. +pub fn refuse_cross_shard_orchestration( + state: &SharedState, + source_collection: &str, + tenant_id: TenantId, + database_id: DatabaseId, + statement: &str, +) -> crate::Result<()> { + if drives_cross_shard_sum(state, source_collection, tenant_id, database_id)? { + return Err(crate::Error::FeatureNotSupported { + detail: format!( + "{statement} into '{source_collection}' moves a materialized sum whose target \ + lives on another vShard, and this route runs it outside a transaction. Run \ + the statement over the SQL or native protocol, which commits the balance \ + with the source rows." + ), + }); + } + Ok(()) +} + /// Resolve one already-extracted join VALUE to its target row's surrogate. /// /// `lookup_surrogate_routed`, never `assign_surrogate_routed`: a join value that @@ -733,4 +788,144 @@ mod tests { other => panic!("plan shape changed: {other:?}"), } } + + /// The fixture pair this module's cross-shard tests rest on. + #[test] + fn the_fixture_target_is_cross_shard() { + assert_ne!( + nodedb_types::CollectionKey::from_bare(DB, "entries").vshard(), + nodedb_types::CollectionKey::from_bare(DB, "accounts").vshard(), + "'accounts' must not share 'entries'' vShard" + ); + } + + /// A staged `INSERT ... SELECT` emits one `PointInsert` per copied row, + /// in the open transaction. The pass the expander runs on them ships each + /// cross-shard balance on a task homed on the target, in that same + /// transaction. The redo's fold table then names no cross-shard target, + /// so the install never folds one on the source's core. + #[tokio::test] + async fn expanded_inserts_ship_cross_shard_balances_and_fold_none() { + let (state, _directory) = test_state(); + declare_binding(&state); + state + .surrogate_assigner + .assign( + nodedb_types::CollectionKey::from_bare(DB, "accounts"), + TENANT, + b"acc-1", + ) + .await + .expect("bind target row"); + let txn = Some(crate::types::TxnId::new(41)); + let mut tasks: Vec = (0..2) + .map(|_| { + let mut task = insert_task("entries", body("acc-1")); + task.txn_id = txn; + task + }) + .collect(); + + let reads = crate::control::server::shared::plan_admission::append_sum_and_period_targets( + &state, + &mut tasks, + TENANT, + DB, + TraceId::ZERO, + ) + .await + .expect("the pass succeeds"); + assert!(reads.is_empty(), "an insert reads no image"); + + let balances: Vec<&PhysicalTask> = tasks + .iter() + .filter(|task| { + matches!( + task.plan, + PhysicalPlan::Document(DocumentOp::ApplyBalanceDelta { .. }) + ) + }) + .collect(); + assert_eq!(balances.len(), 2, "one balance per copied row"); + for balance in balances { + assert_eq!( + balance.vshard_id, + crate::query::sum_target_vshard(DB, "accounts") + ); + assert_eq!(balance.txn_id, txn, "the balance joins the transaction"); + match &balance.plan { + PhysicalPlan::Document(DocumentOp::ApplyBalanceDelta { delta, .. }) => { + assert_eq!(delta, "25"); + } + other => panic!("plan shape changed: {other:?}"), + } + } + + let plans: Vec = tasks.iter().map(|task| task.plan.clone()).collect(); + assert!( + crate::control::wal_replication::transaction_redo::sum_targets::redo_sum_targets( + &plans + ) + .is_empty(), + "the source core folds no cross-shard target" + ); + } + + /// A source that drives a cross-shard target is reported, and the + /// autocommit orchestrators refuse it with the statement and collection + /// named. A collection that drives nothing passes. + #[test] + fn a_cross_shard_source_refuses_orchestration() { + let (state, _directory) = test_state(); + declare_binding(&state); + + assert!(drives_cross_shard_sum(&state, "entries", TENANT, DB).expect("index probe")); + assert!(!drives_cross_shard_sum(&state, "unrelated", TENANT, DB).expect("index probe")); + + match refuse_cross_shard_orchestration(&state, "entries", TENANT, DB, "INSERT ... SELECT") { + Err(crate::Error::FeatureNotSupported { detail }) => { + assert!(detail.contains("INSERT ... SELECT"), "{detail}"); + assert!(detail.contains("'entries'"), "{detail}"); + } + other => panic!("expected FeatureNotSupported, got {other:?}"), + } + refuse_cross_shard_orchestration(&state, "unrelated", TENANT, DB, "MERGE") + .expect("a collection with no binding orchestrates"); + } + + /// A source whose every target shares its vShard rides the source write, + /// so it neither reports a cross-shard sum nor refuses orchestration. + #[test] + fn a_co_resident_source_orchestrates() { + let (state, _directory) = test_state(); + let (source, target) = ("uo_entries", "uo_accounts"); + assert_eq!( + nodedb_types::CollectionKey::from_bare(DB, source).vshard(), + nodedb_types::CollectionKey::from_bare(DB, target).vshard(), + "the pair must share a vShard" + ); + let catalog = state.credentials.catalog(); + let mut stored = StoredCollection::stamped_for_test(TENANT.as_u64(), target, "tester"); + stored.materialized_sums.push(MaterializedSumDef { + target_collection: target.to_string(), + target_column: "balance".to_string(), + source_collection: source.to_string(), + join_column: "account_id".to_string(), + value_expr: nodedb_query::expr::SqlExpr::Column("amount".to_string()), + }); + catalog + .put_collection(DB, &stored) + .expect("persist target collection"); + catalog + .put_collection( + DB, + &StoredCollection::stamped_for_test(TENANT.as_u64(), source, "tester"), + ) + .expect("persist source collection"); + state.materialized_sum_index.invalidate(); + + assert!(!drives_cross_shard_sum(&state, source, TENANT, DB).expect("index probe")); + refuse_cross_shard_orchestration(&state, source, TENANT, DB, "UPDATE ... FROM") + .expect("a co-resident sum orchestrates"); + } } diff --git a/nodedb/src/control/planner/materialized_sum/settle.rs b/nodedb/src/control/planner/materialized_sum/settle.rs index c6c744cde..1f279aef3 100644 --- a/nodedb/src/control/planner/materialized_sum/settle.rs +++ b/nodedb/src/control/planner/materialized_sum/settle.rs @@ -40,13 +40,15 @@ //! the presence of a resolved surrogate is what says "nobody else is applying //! this one". //! -//! That leaves the Control-Plane orchestrators — `MERGE`, `UPDATE ... FROM`, -//! `INSERT ... SELECT` and the staged-transaction expanders — untouched: they -//! resolve their rows through +//! The staged-transaction expanders for `MERGE`, `UPDATE ... FROM` and +//! `INSERT ... SELECT` run this same pass on the point writes they emit. The +//! autocommit orchestrators resolve their rows through //! [`resolve_sum_targets_for_bodies`](super::resolve::resolve_sum_targets_for_bodies) -//! and dispatch through `dispatch_local` without ever reaching this pass, so -//! every join value they resolve stays resolved and the Data Plane keeps -//! applying their deltas exactly as before. +//! and apply on the source's vShard alone, so they refuse a source with a +//! cross-shard target +//! ([`refuse_cross_shard_orchestration`](super::resolve::refuse_cross_shard_orchestration)). +//! The SQL and native protocols run such a statement through the expanders +//! in an implicit transaction instead. //! //! # The images are only as good as the version they were read at //! diff --git a/nodedb/src/control/server/native/dispatch/direct_ops.rs b/nodedb/src/control/server/native/dispatch/direct_ops.rs index 8a2c0d3f9..e7514dd63 100644 --- a/nodedb/src/control/server/native/dispatch/direct_ops.rs +++ b/nodedb/src/control/server/native/dispatch/direct_ops.rs @@ -263,8 +263,9 @@ pub(crate) async fn handle_direct_op( Err(error) => return error_to_native(seq, &crate::Error::from(error)), }; - // A write that fires a BEFORE, INSTEAD OF or SYNC AFTER body, or a - // MERGE into an edge-bearing collection, runs in one transaction, as + // A write that fires a BEFORE, INSTEAD OF or SYNC AFTER body, a MERGE + // into an edge-bearing collection, or a join-expanding write into the + // source of a cross-shard materialized sum, runs in one transaction, as // the SQL path's does. The loop there clone-checks, authorizes and // meters each task itself. if statement_needs_implicit_txn(ctx.state, &tasks) { diff --git a/nodedb/src/control/server/native/dispatch/direct_orchestrated.rs b/nodedb/src/control/server/native/dispatch/direct_orchestrated.rs index 377d2e361..ef055c4c9 100644 --- a/nodedb/src/control/server/native/dispatch/direct_orchestrated.rs +++ b/nodedb/src/control/server/native/dispatch/direct_orchestrated.rs @@ -44,18 +44,22 @@ pub(super) async fn dispatch_orchestrated( PhysicalPlan::Document(nodedb_physical::physical_plan::DocumentOp::InsertSelect { .. }) ) { let task = bare_task(ctx, plan, vshard_id); - let authorized = match super::sql_gateway::authorize_native_task(ctx, &task) { - Ok(authorized) => authorized, - Err(error) => return Some(error_to_native(seq, &error)), - }; - let _request = ctx.state.tenant_request_guard(tenant_id); - let result = - crate::control::insert_select::run_authorized_insert_select(ctx.state, authorized) - .await; - return Some(match result { - Ok(resp) => data_plane_response_to_native(ctx, seq, plan, &resp), - Err(e) => error_to_native(seq, &e), - }); + // A copy into the source of a cross-shard materialized sum runs in the + // implicit transaction, which ships each balance to its target. + if !statement_needs_implicit_txn(ctx.state, std::slice::from_ref(&task)) { + let authorized = match super::sql_gateway::authorize_native_task(ctx, &task) { + Ok(authorized) => authorized, + Err(error) => return Some(error_to_native(seq, &error)), + }; + let _request = ctx.state.tenant_request_guard(tenant_id); + let result = + crate::control::insert_select::run_authorized_insert_select(ctx.state, authorized) + .await; + return Some(match result { + Ok(resp) => data_plane_response_to_native(ctx, seq, plan, &resp), + Err(e) => error_to_native(seq, &e), + }); + } } // Autocommit `MERGE` orchestrates on the Control Plane; never reaches the @@ -68,8 +72,10 @@ pub(super) async fn dispatch_orchestrated( }) ) { let task = bare_task(ctx, plan, vshard_id); - // A MERGE into an edge-bearing collection runs in the implicit - // transaction, which stages each removed row's edge tasks. + // A MERGE into an edge-bearing collection, or into the source of a + // cross-shard materialized sum, runs in the implicit transaction. It + // stages each removed row's edge tasks and ships each balance to its + // target. if !statement_needs_implicit_txn(ctx.state, std::slice::from_ref(&task)) { let authorized = match super::sql_gateway::authorize_native_task(ctx, &task) { Ok(authorized) => authorized, @@ -96,20 +102,24 @@ pub(super) async fn dispatch_orchestrated( }) ) { let task = bare_task(ctx, plan, vshard_id); - let authorized = match super::sql_gateway::authorize_native_task(ctx, &task) { - Ok(authorized) => authorized, - Err(error) => return Some(error_to_native(seq, &error)), - }; - let _request = ctx.state.tenant_request_guard(tenant_id); - let result = - crate::control::update_from_join_orchestrator::run_authorized_update_from_join( - ctx.state, authorized, - ) - .await; - return Some(match result { - Ok(resp) => data_plane_response_to_native(ctx, seq, plan, &resp), - Err(e) => error_to_native(seq, &e), - }); + // An update of the source of a cross-shard materialized sum runs in the + // implicit transaction, which ships each balance to its target. + if !statement_needs_implicit_txn(ctx.state, std::slice::from_ref(&task)) { + let authorized = match super::sql_gateway::authorize_native_task(ctx, &task) { + Ok(authorized) => authorized, + Err(error) => return Some(error_to_native(seq, &error)), + }; + let _request = ctx.state.tenant_request_guard(tenant_id); + let result = + crate::control::update_from_join_orchestrator::run_authorized_update_from_join( + ctx.state, authorized, + ) + .await; + return Some(match result { + Ok(resp) => data_plane_response_to_native(ctx, seq, plan, &resp), + Err(e) => error_to_native(seq, &e), + }); + } } // A governed predicate resolves to a concrete row set before proposing — see diff --git a/nodedb/src/control/server/native/dispatch/sql_planned.rs b/nodedb/src/control/server/native/dispatch/sql_planned.rs index fa2d3eadf..7ba94e01b 100644 --- a/nodedb/src/control/server/native/dispatch/sql_planned.rs +++ b/nodedb/src/control/server/native/dispatch/sql_planned.rs @@ -143,8 +143,9 @@ pub(super) async fn execute_planned( } // A statement outside a transaction block whose writes fire a BEFORE, - // INSTEAD OF or SYNC AFTER body, or a MERGE into an edge-bearing - // collection, runs in an implicit transaction ahead of every autocommit + // INSTEAD OF or SYNC AFTER body, a MERGE into an edge-bearing collection, + // or a join-expanding write into the source of a cross-shard + // materialized sum, runs in an implicit transaction ahead of every autocommit // route: its writes stage on their vShards' leaders and commit together // at its end. let in_txn_block = ctx.sessions.transaction_state(ctx.peer_addr) == TransactionState::InBlock; diff --git a/nodedb/src/control/server/pgwire/handler/routing/execute.rs b/nodedb/src/control/server/pgwire/handler/routing/execute.rs index 35bdf6000..8ed955abb 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/execute.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/execute.rs @@ -173,8 +173,10 @@ impl NodeDbPgHandler { let effective_schema = Some(&effective_schema_owned); // A statement outside a transaction block whose writes fire a - // BEFORE, INSTEAD OF or SYNC AFTER body, or a MERGE into an - // edge-bearing collection, runs in an implicit transaction, + // BEFORE, INSTEAD OF or SYNC AFTER body, a MERGE into an + // edge-bearing collection, or an INSERT ... SELECT, UPDATE ... FROM + // or MERGE into the source of a cross-shard materialized sum, + // runs in an implicit transaction, // ahead of every autocommit route: // its writes stage on their vShards' leaders and commit together // at its end. diff --git a/nodedb/src/control/server/shared/plan_admission.rs b/nodedb/src/control/server/shared/plan_admission.rs index 83a578681..46577c02f 100644 --- a/nodedb/src/control/server/shared/plan_admission.rs +++ b/nodedb/src/control/server/shared/plan_admission.rs @@ -181,7 +181,27 @@ pub async fn append_derived_tasks( trace_id, ) .await?; + append_sum_and_period_targets(state, tasks, tenant_id, database_id, trace_id).await +} +/// Resolve every write's materialized-sum targets, append one +/// `ApplyBalanceDelta` task per cross-shard balance, and resolve every +/// write's period-lock reference row. +/// +/// A plain statement runs this through [`append_derived_tasks`]. The +/// in-transaction expanders run it on the point writes they emit. So every +/// write that moves a sum source ships its cross-shard balance on a task +/// homed on the target's vShard. +/// +/// Returns the read-set entries covering the images every cross-shard +/// balance was settled from. The caller unions them into its read set. +pub async fn append_sum_and_period_targets( + state: &SharedState, + tasks: &mut Vec, + tenant_id: crate::types::TenantId, + database_id: crate::types::DatabaseId, + trace_id: TraceId, +) -> crate::Result> { let sum_target_reads = crate::control::planner::materialized_sum::resolve_materialized_sum_targets( state, diff --git a/nodedb/src/control/server/shared/session/expander_stage.rs b/nodedb/src/control/server/shared/session/expander_stage.rs index a03666cdc..bce388db5 100644 --- a/nodedb/src/control/server/shared/session/expander_stage.rs +++ b/nodedb/src/control/server/shared/session/expander_stage.rs @@ -17,6 +17,10 @@ use super::store::SessionStore; use crate::bridge::envelope::{PhysicalPlan, Response}; use crate::control::insert_select::resolve_and_emit_insert_select_ops; use crate::control::merge_orchestrator::resolve_and_emit_merge_ops; +use crate::control::planner::calvin::write_class::{ + plan_counts_toward_statement_tag, plans_have_user_write, +}; +use crate::control::server::shared::plan_admission::append_sum_and_period_targets; use crate::control::state::SharedState; use crate::control::update_from_join_orchestrator::resolve_and_emit_update_from_join_ops; use nodedb_physical::physical_plan::DocumentOp; @@ -61,7 +65,8 @@ where /// every other plan, and outside a transaction block. /// /// The ops are `PointInsert` for an inserted row, `PointPut` for an updated -/// row's post-image, and `PointDelete` for a removed row. +/// row's post-image, `PointDelete` for a removed row, and an +/// `ApplyBalanceDelta` for each cross-shard materialized-sum balance they move. pub(crate) async fn expand_in_tx( state: &SharedState, sessions: &SessionStore, @@ -72,7 +77,7 @@ pub(crate) async fn expand_in_tx( if sessions.transaction_state(session_id) != TransactionState::InBlock { return Ok(None); } - let expanded = match &task.plan { + let (mut ops, kind) = match &task.plan { PhysicalPlan::Document(DocumentOp::Merge { resolved_inserts: None, .. @@ -105,13 +110,32 @@ pub(crate) async fn expand_in_tx( } // A `BatchInsert` page is autocommit-shaped; only point ops give an overlay // post-image, per-row undo, and row-level redo, so expand back to points. - PhysicalPlan::Document(DocumentOp::BatchInsert { .. }) => ( - expand_batch_insert(task).map_err(StagingGateError::Dispatch)?, - StagedTagKind::Insert, - ), + // The page carries the statement's resolution and deferrals already. + PhysicalPlan::Document(DocumentOp::BatchInsert { .. }) => { + return Ok(Some(( + expand_batch_insert(task).map_err(StagingGateError::Dispatch)?, + StagedTagKind::Insert, + ))); + } _ => return Ok(None), }; - Ok(Some(expanded)) + // The expanded point writes never passed statement admission. They take + // its sum and period-lock pass here. A cross-shard balance ships on an + // `ApplyBalanceDelta` task homed on the target's vShard, and the source + // write does not fold it. + let reads = append_sum_and_period_targets( + state, + &mut ops, + task.tenant_id, + task.database_id, + crate::types::TraceId::ZERO, + ) + .await + .map_err(StagingGateError::Dispatch)?; + // The images each shipped balance was settled from join the read set, so + // COMMIT aborts when a concurrent write moved them. + sessions.record_read_entries(session_id, reads); + Ok(Some((ops, kind))) } /// Expand a `BatchInsert` page into one `PointInsert` op per row. Nothing is @@ -186,8 +210,12 @@ where { // Each `stage_write` dispatches into the overlay AND buffers the concrete op // for COMMIT replay — the raw Merge/UpdateFromJoin/InsertSelect is never buffered. + // A balance task the expansion appended stages too, but its count describes + // a row the statement never named. + let has_user_write = plans_have_user_write(ops.iter().map(|op| &op.plan)); let mut affected = 0usize; for op in ops { + let counts = plan_counts_toward_statement_tag(&op.plan, has_user_write); // A removed row of an edge-bearing collection carries its node's edge // tasks, read before the removal stages. let node_delete_tasks = @@ -199,7 +227,9 @@ where .await .map_err(StagingGateError::Dispatch)?; let outcome = stage_write(state, sessions, session_id, op, &dispatch).await?; - affected += outcome.affected; + if counts { + affected += outcome.affected; + } super::node_delete_stage::stage_node_delete_tasks( state, sessions, diff --git a/nodedb/src/control/server/shared/txn_route/joined.rs b/nodedb/src/control/server/shared/txn_route/joined.rs index 4d4e14530..54610c891 100644 --- a/nodedb/src/control/server/shared/txn_route/joined.rs +++ b/nodedb/src/control/server/shared/txn_route/joined.rs @@ -44,15 +44,78 @@ pub fn statement_fires_joined_body(state: &SharedState, tasks: &[PhysicalTask]) } /// Whether a statement outside a transaction block runs in an implicit -/// transaction. It does when it fires a joined body, and when it is a -/// `MERGE` into a collection an edge was ever written into: the MERGE then -/// stages each removed row with its node's edge tasks, and COMMIT applies -/// them together. +/// transaction. It does in three cases: +/// +/// - It fires a joined body. +/// - It is a `MERGE` into a collection an edge was ever written into. The +/// MERGE then stages each removed row with its node's edge tasks, and +/// COMMIT applies them together. +/// - It is an `INSERT ... SELECT`, `UPDATE ... FROM` or `MERGE` that writes +/// the source of a materialized sum whose target lives on another vShard. +/// The expansion ships that balance on an `ApplyBalanceDelta` task homed +/// on the target, and COMMIT applies it with the source rows. pub fn statement_needs_implicit_txn(state: &SharedState, tasks: &[PhysicalTask]) -> bool { statement_fires_joined_body(state, tasks) || tasks .iter() - .any(|task| merges_into_edge_bearing(state, task)) + .any(|task| merges_into_edge_bearing(state, task) || moves_cross_shard_sum(state, task)) +} + +/// Whether `task` is an unexpanded `INSERT ... SELECT`, `UPDATE ... FROM` or +/// `MERGE` whose written collection drives a materialized sum with a +/// cross-shard target. Its orchestrator applies on the written collection's +/// vShard alone, so it cannot move that target. A catalog read error reads +/// as a cross-shard sum, so the statement takes the transaction. +fn moves_cross_shard_sum(state: &SharedState, task: &PhysicalTask) -> bool { + let PhysicalPlan::Document(op) = &task.plan else { + return false; + }; + let written = match op { + DocumentOp::InsertSelect { + target_collection, .. + } + | DocumentOp::UpdateFromJoin { + target_collection, + source_rows: None, + .. + } + | DocumentOp::Merge { + target_collection, + resolved_inserts: None, + .. + } => target_collection.as_str(), + DocumentOp::UpdateFromJoin { .. } + | DocumentOp::Merge { .. } + | DocumentOp::PointGet { .. } + | DocumentOp::PointPut { .. } + | DocumentOp::PointInsert { .. } + | DocumentOp::PointDelete { .. } + | DocumentOp::PointUpdate { .. } + | DocumentOp::Upsert { .. } + | DocumentOp::BatchInsert { .. } + | DocumentOp::BulkUpdate { .. } + | DocumentOp::BulkDelete { .. } + | DocumentOp::Truncate { .. } + | DocumentOp::Scan { .. } + | DocumentOp::RangeScan { .. } + | DocumentOp::Register { .. } + | DocumentOp::IndexLookup { .. } + | DocumentOp::IndexedFetch { .. } + | DocumentOp::DropIndex { .. } + | DocumentOp::BackfillIndex { .. } + | DocumentOp::EstimateCount { .. } + | DocumentOp::MaterializeScan { .. } + | DocumentOp::ResolveWrite(_) + | DocumentOp::ResolvedWrite { .. } + | DocumentOp::ApplyBalanceDelta { .. } => return false, + }; + crate::control::planner::materialized_sum::drives_cross_shard_sum( + state, + written, + task.tenant_id, + task.database_id, + ) + .unwrap_or(true) } /// Whether `task` is a `MERGE` into an edge-bearing collection. A catalog diff --git a/nodedb/src/control/server/shared/txn_route/route.rs b/nodedb/src/control/server/shared/txn_route/route.rs index 281f8e7d4..7501c0d5f 100644 --- a/nodedb/src/control/server/shared/txn_route/route.rs +++ b/nodedb/src/control/server/shared/txn_route/route.rs @@ -28,6 +28,9 @@ use nodedb_physical::physical_task::PhysicalTask; use crate::bridge::envelope::{PhysicalPlan, Response}; use crate::control::lease::QueryLeaseScope; +use crate::control::planner::calvin::write_class::{ + plan_counts_toward_statement_tag, plans_have_user_write, +}; use crate::control::security::auth_context::AuthContext; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::response_shape::types::{PlanKind, describe_plan}; @@ -182,17 +185,26 @@ where F: Fn(PhysicalTask) -> Fut, Fut: Future>, { + // A balance task the expansion appended stages like any op, but its count + // describes a row the statement never named. + let has_user_write = plans_have_user_write(ops.iter().map(|op| &op.plan)); let mut affected = 0usize; let mut returning_rows = Vec::new(); for op in ops { + let counts = plan_counts_toward_statement_tag(&op.plan, has_user_write); match route_point(ctx, op, expanded, dispatch).await? { TxnTaskOutcome::Staged(outcome) => { - affected += outcome.affected; + if counts { + affected += outcome.affected; + } returning_rows.extend(outcome.returning_rows); } TxnTaskOutcome::CloneHandled(resp) => { - affected += require_affected_count(resp.payload.as_ref()) + let changed = require_affected_count(resp.payload.as_ref()) .map_err(StagingGateError::Dispatch)? as usize; + if counts { + affected += changed; + } } TxnTaskOutcome::InsteadOf => {} TxnTaskOutcome::Buffered | TxnTaskOutcome::Dispatch(_) => { diff --git a/nodedb/src/control/server/shared/write_admission/admission_keys.rs b/nodedb/src/control/server/shared/write_admission/admission_keys.rs index d4d00001e..469be6e1f 100644 --- a/nodedb/src/control/server/shared/write_admission/admission_keys.rs +++ b/nodedb/src/control/server/shared/write_admission/admission_keys.rs @@ -507,6 +507,60 @@ mod tests { assert!(!table.try_acquire(TxnId::new(TxnId::AUTOCOMMIT_EPOCH, 2), truncate_keys)); } + /// A fast-path point UPDATE of a balance target row and the Calvin + /// transaction that moves the row's balance lock the same row key + /// `Exclusive`. While the transaction holds its locks, from the grant + /// that stages the balance's post-image to the install, the UPDATE + /// cannot take the key. So it lands before the stage or after the + /// install, and the install never overwrites it. + #[test] + fn a_point_update_of_a_balance_target_is_fenced_by_the_balance_write() { + let balance = PhysicalPlan::Document(DocumentOp::ApplyBalanceDelta { + collection: qualified("accts"), + document_id: "00000009".to_owned(), + surrogate: Surrogate::new(9), + column: "balance".to_owned(), + delta: "5".to_owned(), + join_column: "account_id".to_owned(), + join_value: "acc".to_owned(), + declared_primary_key: None, + }); + let update = PhysicalPlan::Document(DocumentOp::PointUpdate { + collection: qualified("accts"), + document_id: "acc".to_owned(), + surrogate: Some(Surrogate::new(9)), + pk_bytes: b"acc".to_vec(), + updates: Vec::new(), + returning: None, + rls_filters: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::NoPolicyApplies, + resolved_sum_targets: Vec::new(), + declared_primary_key: None, + }); + + // The scheduler expands a transaction's write keys by the same rule. + let mut calvin_keys = WriteKeys::default(); + add_plan_write_keys(&mut calvin_keys, &balance).expect("a balance write has keys"); + let held = expand_write_key_sets(&calvin_keys.into_key_sets()); + let (requested, sequenced) = keys_of(&update); + assert!(sequenced); + assert_eq!( + held.get(&surrogate_key("accts", 9)), + Some(&LockMode::Exclusive) + ); + assert_eq!( + requested.get(&surrogate_key("accts", 9)), + Some(&LockMode::Exclusive) + ); + + let mut table = LockManager::new(); + assert!(table.try_acquire(TxnId::new(1, 0), held)); + assert!( + !table.try_acquire(TxnId::new(TxnId::AUTOCOMMIT_EPOCH, 0), requested), + "the UPDATE must not take the row while the balance write holds it" + ); + } + /// A committed redo locks every collection it writes `Exclusive`. #[test] fn a_whole_collection_request_locks_each_collection_exclusive() { diff --git a/nodedb/src/control/update_from_join_orchestrator/expand_staged_update_from_join.rs b/nodedb/src/control/update_from_join_orchestrator/expand_staged_update_from_join.rs index a5f9d253d..b165f24c1 100644 --- a/nodedb/src/control/update_from_join_orchestrator/expand_staged_update_from_join.rs +++ b/nodedb/src/control/update_from_join_orchestrator/expand_staged_update_from_join.rs @@ -19,7 +19,9 @@ use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; /// Resolve one in-transaction `UpdateFromJoin` task into the concrete, /// surrogate-carrying `PointPut` tasks its matched target rows expand to. -/// `task.txn_id` must be the active transaction. Caller stages + buffers each op. +/// `task.txn_id` must be the active transaction. The ops carry no sum or +/// period-lock resolution: the caller runs the statement pass on them, then +/// stages + buffers each one. pub(crate) async fn resolve_and_emit_update_from_join_ops( state: &SharedState, tenant_id: TenantId, @@ -72,23 +74,6 @@ pub(crate) async fn resolve_and_emit_update_from_join_ops( )? .vshard(); - // A join-column rewrite debits the target left and credits the one joined — - // resolving post-images alone would leave the abandoned target overstated. - let sum_bodies: Vec<&[u8]> = resolved - .iter() - .flat_map(|(_, _, body, old_body)| [body.as_slice(), old_body.as_slice()]) - .collect(); - let resolved_sum_targets = - crate::control::planner::materialized_sum::resolve_sum_targets_for_bodies( - state, - &sum_bodies, - target_collection.as_str(), - tenant_id, - task.database_id, - crate::types::TraceId::ZERO, - ) - .await?; - let mut out: Vec = Vec::with_capacity(resolved.len()); for (_doc_id, surrogate_u32, body, _old_body) in resolved { // The wire surrogate is never absent: every matched `UPDATE ... FROM` @@ -109,7 +94,10 @@ pub(crate) async fn resolve_and_emit_update_from_join_ops( // The op owns the statement's projection; expanded puts answer no client. returning: None, rls_filters: Vec::new(), - resolved_sum_targets: resolved_sum_targets.clone(), + // Filled by the caller's sum and period-lock pass. It reads + // the stored row, so a join-column rewrite debits the target + // the row leaves as well as crediting the one it joins. + resolved_sum_targets: Vec::new(), }), post_set_op: PostSetOp::None, txn_id: task.txn_id, diff --git a/nodedb/src/control/update_from_join_orchestrator/orchestrator.rs b/nodedb/src/control/update_from_join_orchestrator/orchestrator.rs index 7bb6b0e50..951d6baa4 100644 --- a/nodedb/src/control/update_from_join_orchestrator/orchestrator.rs +++ b/nodedb/src/control/update_from_join_orchestrator/orchestrator.rs @@ -105,6 +105,15 @@ pub(crate) async fn run_update_from_join( state: &SharedState, args: UpdateFromJoinArgs<'_>, ) -> crate::Result { + // The write applies on the target's vShard alone, so it folds only + // co-resident balances. + crate::control::planner::materialized_sum::refuse_cross_shard_orchestration( + state, + args.target_collection, + args.tenant_id, + args.database_id, + "UPDATE ... FROM", + )?; // Checked once: a target driving no materialized-sum binding, declaring // no period lock AND carrying no write policy skips the RESOLVE round // trip and retry loop entirely. A write policy needs the matched diff --git a/nodedb/src/data/executor/enforcement/materialized_sum/apply.rs b/nodedb/src/data/executor/enforcement/materialized_sum/apply.rs index 699ed597d..6304360a2 100644 --- a/nodedb/src/data/executor/enforcement/materialized_sum/apply.rs +++ b/nodedb/src/data/executor/enforcement/materialized_sum/apply.rs @@ -43,11 +43,12 @@ //! ([`sum_target_is_co_resident`](crate::query::sum_target_is_co_resident)) //! that exists so the two answers cannot differ. //! -//! A cross-shard target the plan DOES resolve is still applied here: the -//! Control-Plane orchestrators (`MERGE`, `UPDATE ... FROM`, `INSERT ... SELECT`, -//! and the staged-transaction expanders) resolve their rows and dispatch their -//! own concrete work without appending a sibling balance task, so for them the -//! resolution still means "this transaction owns it". +//! No write path hands this core a cross-shard target it must apply. Plain +//! writes and the in-transaction expanders take the same Control-Plane pass, +//! which ships every cross-shard balance on its own task. The autocommit +//! orchestrators (`MERGE`, `UPDATE ... FROM`, `INSERT ... SELECT`) refuse a +//! source that drives a cross-shard target, and the SQL and native protocols +//! run such a statement through the expanders instead. //! //! # Identity comes from the plan, never from a store probe //! @@ -174,11 +175,7 @@ impl CoreLoop { // collection consults. // // The absence of the resolution IS the instruction; nothing is - // re-derived. A cross-shard target the plan DID resolve was - // resolved by a Control-Plane orchestrator that ships no - // sibling task — `MERGE`, `UPDATE ... FROM`, `INSERT ... SELECT` - // and the staged-transaction expanders all dispatch their own - // concrete work — so it is still this transaction's to apply. + // re-derived. if !co_resident && resolved_target(ctx, &binding.target_collection, &join_value).is_none() { diff --git a/nodedb/tests/wire/cases/materialized_sum_cross_shard_expanders.rs b/nodedb/tests/wire/cases/materialized_sum_cross_shard_expanders.rs new file mode 100644 index 000000000..01a657395 --- /dev/null +++ b/nodedb/tests/wire/cases/materialized_sum_cross_shard_expanders.rs @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `INSERT ... SELECT`, `UPDATE ... FROM` and `MERGE` into the source of a +//! materialized sum whose target lives on another vShard. +//! +//! Each statement expands into point writes on the source. A balance on a +//! cross-shard target cannot fold on the source's core: that core owns no +//! target row. Each expansion ships the balance on its own task, homed on the +//! target's vShard, and COMMIT applies it with the source rows. +//! +//! The server runs two Data Plane cores, and the pair below lands on +//! different cores. A balance folded on the source's core then finds no +//! target row. + +use nodedb_types::id::DatabaseId; +use tokio_postgres::SimpleQueryMessage; + +use crate::harness::TestServer; + +/// The sum's target collection. +const ACCOUNTS: &str = "ptv_acct"; +/// The sum's source collection. +const POSTINGS: &str = "ptv_post"; +/// Rows the statements copy or join from. Its home does not matter. +const FEED: &str = "ptv_feed"; +/// Data Plane cores the server runs. +const CORES: usize = 2; + +fn vshard(name: &str) -> u32 { + nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, name) + .vshard() + .as_u32() +} + +/// The premise every test rests on: the target and the source hash to +/// different vShards, and those vShards live on different cores. +#[test] +fn the_sum_pair_spans_two_vshards_and_two_cores() { + assert_ne!( + vshard(ACCOUNTS), + vshard(POSTINGS), + "the pair must plan a separate balance task" + ); + assert_ne!( + vshard(ACCOUNTS) as usize % CORES, + vshard(POSTINGS) as usize % CORES, + "a balance folded on the source's core must find no target row" + ); +} + +/// Every `CommandComplete` count in `sql`'s simple-query response, in wire +/// order. +async fn command_counts(server: &TestServer, sql: &str) -> Vec { + let messages = server + .client + .simple_query(sql) + .await + .unwrap_or_else(|e| panic!("run {sql}: {e:?}")); + messages + .into_iter() + .filter_map(|m| match m { + SimpleQueryMessage::CommandComplete(n) => Some(n), + _ => None, + }) + .collect() +} + +async fn exec(server: &TestServer, sql: &str) { + server + .exec(sql) + .await + .unwrap_or_else(|e| panic!("run {sql}: {e}")); +} + +/// Start a server and create the sum pair and the feed. Accounts `acc` and +/// `acc2` start empty. Posting `e0` puts 10 on `acc`, posting `e1` puts 5 on +/// `acc`. +async fn sum_fixture() -> TestServer { + let server = TestServer::start_multicores(CORES).await; + exec( + &server, + &format!( + "CREATE COLLECTION {ACCOUNTS} (id TEXT PRIMARY KEY, owner TEXT) \ + WITH (engine='document_strict')" + ), + ) + .await; + exec( + &server, + &format!( + "CREATE COLLECTION {POSTINGS} (id TEXT PRIMARY KEY, account_id TEXT, amount TEXT) \ + WITH (engine='document_strict')" + ), + ) + .await; + exec( + &server, + &format!( + "ALTER COLLECTION {ACCOUNTS} ADD COLUMN balance TEXT \ + MATERIALIZED_SUM SOURCE {POSTINGS} \ + ON {POSTINGS}.account_id = {ACCOUNTS}.id VALUE {POSTINGS}.amount" + ), + ) + .await; + exec( + &server, + &format!( + "CREATE COLLECTION {FEED} (id TEXT PRIMARY KEY, account_id TEXT, amount TEXT) \ + WITH (engine='document_strict')" + ), + ) + .await; + for account in ["acc", "acc2"] { + exec( + &server, + &format!( + "INSERT INTO {ACCOUNTS} (id, owner, balance) VALUES ('{account}', 'alice', '0')" + ), + ) + .await; + } + for (id, amount) in [("e0", "10"), ("e1", "5")] { + exec( + &server, + &format!( + "INSERT INTO {POSTINGS} (id, account_id, amount) VALUES ('{id}', 'acc', '{amount}')" + ), + ) + .await; + } + assert_eq!( + balance(&server, "acc").await, + "15", + "the seed postings land" + ); + server +} + +/// Insert one feed row. +async fn feed(server: &TestServer, id: &str, account: &str, amount: &str) { + exec( + server, + &format!( + "INSERT INTO {FEED} (id, account_id, amount) VALUES ('{id}', '{account}', '{amount}')" + ), + ) + .await; +} + +/// The stored balance of `account`. +async fn balance(server: &TestServer, account: &str) -> String { + let rows = server + .query_text(&format!( + "SELECT balance FROM {ACCOUNTS} WHERE id = '{account}'" + )) + .await + .unwrap_or_else(|e| panic!("read the balance of {account}: {e}")); + match rows.as_slice() { + [value] => value.clone(), + other => panic!("one balance row for {account}, got {other:?}"), + } +} + +const INSERT_SELECT: &str = "INSERT INTO ptv_post (id, account_id, amount) \ + SELECT id, account_id, amount FROM ptv_feed"; + +/// The amount of `e0` changes in place. `e1` moves from `acc` to `acc2`. +const UPDATE_FROM: &str = "UPDATE ptv_post SET amount = f.amount, account_id = f.account_id \ + FROM ptv_feed f WHERE ptv_post.id = f.id"; + +/// Seed the feed `UPDATE_FROM` joins. +async fn update_feed(server: &TestServer) { + feed(server, "e0", "acc", "25").await; + feed(server, "e1", "acc2", "5").await; +} + +/// `BEGIN; INSERT ... SELECT; COMMIT` credits each copied row's amount to its +/// cross-shard account, and tags only the copied rows. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_staged_insert_select_moves_a_cross_shard_sum() { + let server = sum_fixture().await; + feed(&server, "f1", "acc", "4").await; + feed(&server, "f2", "acc2", "6").await; + + assert_eq!( + command_counts(&server, &format!("BEGIN; {INSERT_SELECT}; COMMIT")).await, + vec![0, 2, 0], + "BEGIN, INSERT 0 2, COMMIT: a balance task adds no count" + ); + + assert_eq!(balance(&server, "acc").await, "19", "10 + 5 + 4"); + assert_eq!(balance(&server, "acc2").await, "6"); +} + +/// `BEGIN; UPDATE ... FROM; COMMIT` moves a cross-shard balance by the +/// amount's difference, and moves a whole posting between accounts when the +/// join column changes. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_staged_update_from_moves_a_cross_shard_sum() { + let server = sum_fixture().await; + update_feed(&server).await; + + assert_eq!( + command_counts(&server, &format!("BEGIN; {UPDATE_FROM}; COMMIT")).await, + vec![0, 2, 0], + "BEGIN, UPDATE 2, COMMIT: a balance task adds no count" + ); + + assert_eq!(balance(&server, "acc").await, "25", "e0 at 25, e1 left"); + assert_eq!(balance(&server, "acc2").await, "5", "e1 joined"); +} + +/// `BEGIN; MERGE; COMMIT` moves a cross-shard balance for its UPDATE arm and +/// its INSERT arm alike. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_staged_merge_moves_a_cross_shard_sum() { + let server = sum_fixture().await; + feed(&server, "e0", "acc", "30").await; + feed(&server, "f1", "acc2", "7").await; + + exec( + &server, + &format!( + "BEGIN; \ + MERGE INTO {POSTINGS} p USING {FEED} f ON p.id = f.id \ + WHEN MATCHED THEN UPDATE SET amount = f.amount \ + WHEN NOT MATCHED THEN INSERT (id, account_id, amount) \ + VALUES (f.id, f.account_id, f.amount); \ + COMMIT" + ), + ) + .await; + + assert_eq!(balance(&server, "acc").await, "35", "e0 at 30, e1 at 5"); + assert_eq!(balance(&server, "acc2").await, "7", "f1 inserted"); +} + +/// Outside a transaction block the same statements run in an implicit +/// transaction, and move the cross-shard balance the same way. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn autocommit_insert_select_and_update_from_move_a_cross_shard_sum() { + let server = sum_fixture().await; + feed(&server, "f1", "acc2", "6").await; + + assert_eq!( + command_counts(&server, INSERT_SELECT).await, + vec![1], + "INSERT 0 1: a balance task adds no count" + ); + assert_eq!(balance(&server, "acc2").await, "6", "f1 copied"); + + exec(&server, &format!("DELETE FROM {FEED} WHERE id = 'f1'")).await; + update_feed(&server).await; + assert_eq!( + command_counts(&server, UPDATE_FROM).await, + vec![2], + "UPDATE 2: a balance task adds no count" + ); + + assert_eq!(balance(&server, "acc").await, "25", "e0 at 25, e1 left"); + assert_eq!(balance(&server, "acc2").await, "11", "f1 and e1"); +} diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index eb9e275cf..e7b102d2d 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -127,6 +127,7 @@ mod kv_sql_select; mod kv_transfer_decimal; mod kv_write_row_level_security; mod kv_write_row_level_security_atomics; +mod materialized_sum_cross_shard_expanders; mod materialized_sum_txn_command_tag; mod merge_insert_atomicity; mod merge_insert_cross_engine_visibility; From af32618241ae8e09a182c5575199466c226cf07a Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sat, 10 Oct 2026 11:41:50 +0800 Subject: [PATCH 09/19] refactor(server): share one atomic statement path across protocols Move statement execution into a shared statement_exec module so native, pgwire and the HTTP routes (/v1/query, /v1/query/stream and WebSocket RPC) run a statement through the same implicit transaction, implicit-edge gate and Calvin commit. Statements the shared router does not take fall back to each route's per-task loop. A MERGE, UPDATE ... FROM or INSERT ... SELECT into a source that drives a cross-shard materialized sum no longer needs the autocommit orchestrators to refuse it: every protocol runs it through the expanders in an implicit transaction. Remove the refusal and the native sql_dispatch_task module it served. Materialized-sum and period-lock writes in a transaction block now read their pre-image as the transaction sees it, so a staged insert or rewrite contributes the correct balance delta. Add wire cases for the HTTP routes and for the in-transaction pre-image. --- .../src/control/insert_select/orchestrator.rs | 9 - .../merge_orchestrator/orchestrator.rs | 9 - .../control/planner/materialized_sum/mod.rs | 4 +- .../planner/materialized_sum/predicate.rs | 7 + .../control/planner/materialized_sum/recon.rs | 35 +- .../planner/materialized_sum/resolve.rs | 88 ++-- .../planner/materialized_sum/settle.rs | 26 +- .../planner/materialized_sum/stored.rs | 36 +- .../src/control/planner/period_lock/lookup.rs | 7 +- .../planner/period_lock/point_update.rs | 1 + .../control/planner/period_lock/predicate.rs | 1 + .../control/planner/period_lock/resolve.rs | 11 +- .../control/planner/period_lock/singular.rs | 13 +- .../procedural/executor/core/dispatch.rs | 18 + .../server/http/routes/atomic_statement.rs | 100 +++++ nodedb/src/control/server/http/routes/mod.rs | 1 + .../http/routes/query/materialized/append.rs | 34 +- .../http/routes/query/materialized/mod.rs | 5 +- .../routes/query/materialized/orchestrated.rs | 4 +- .../http/routes/query/materialized/request.rs | 37 ++ .../server/http/routes/query/ndjson.rs | 98 +++-- .../server/http/routes/ws_rpc/execute_sql.rs | 100 +++-- .../server/native/dispatch/cluster_array.rs | 6 +- .../server/native/dispatch/direct_ops.rs | 2 + .../server/native/dispatch/edge_recon_gate.rs | 98 +---- .../src/control/server/native/dispatch/mod.rs | 1 - .../server/native/dispatch/single_task.rs | 6 +- .../native/dispatch/sql_dispatch_task.rs | 142 ------- .../server/native/dispatch/sql_fold.rs | 194 +++------ .../server/native/dispatch/sql_gateway.rs | 98 +---- .../server/native/dispatch/sql_loop.rs | 399 +++--------------- .../server/native/dispatch/sql_planned.rs | 224 ++++------ .../server/pgwire/handler/routing/execute.rs | 5 + .../server/shared/cluster_array_dispatch.rs | 7 +- .../neutral/collection/dml/parse/dispatch.rs | 6 +- nodedb/src/control/server/shared/mod.rs | 1 + .../control/server/shared/plan_admission.rs | 46 +- .../server/shared/session/expander_stage.rs | 5 +- .../control/server/shared/session/read_set.rs | 21 +- .../server/shared/statement_exec/answer.rs | 190 +++++++++ .../server/shared/statement_exec/atomic.rs | 233 ++++++++++ .../server/shared/statement_exec/dispatch.rs | 246 +++++++++++ .../shared/statement_exec/edge_recon.rs | 85 ++++ .../server/shared/statement_exec/error.rs | 53 +++ .../server/shared/statement_exec/mod.rs | 21 + .../server/shared/statement_exec/task_loop.rs | 297 +++++++++++++ nodedb/src/control/system_txn/tests.rs | 1 + .../orchestrator.rs | 9 - .../enforcement/materialized_sum/apply.rs | 8 +- .../materialized_sum_cross_shard_expanders.rs | 8 +- .../materialized_sum_cross_shard_http.rs | 177 ++++++++ .../cases/materialized_sum_txn_preimage.rs | 208 +++++++++ nodedb/tests/wire/cases/mod.rs | 2 + 53 files changed, 2231 insertions(+), 1212 deletions(-) create mode 100644 nodedb/src/control/server/http/routes/atomic_statement.rs delete mode 100644 nodedb/src/control/server/native/dispatch/sql_dispatch_task.rs create mode 100644 nodedb/src/control/server/shared/statement_exec/answer.rs create mode 100644 nodedb/src/control/server/shared/statement_exec/atomic.rs create mode 100644 nodedb/src/control/server/shared/statement_exec/dispatch.rs create mode 100644 nodedb/src/control/server/shared/statement_exec/edge_recon.rs create mode 100644 nodedb/src/control/server/shared/statement_exec/error.rs create mode 100644 nodedb/src/control/server/shared/statement_exec/mod.rs create mode 100644 nodedb/src/control/server/shared/statement_exec/task_loop.rs create mode 100644 nodedb/tests/wire/cases/materialized_sum_cross_shard_http.rs create mode 100644 nodedb/tests/wire/cases/materialized_sum_txn_preimage.rs diff --git a/nodedb/src/control/insert_select/orchestrator.rs b/nodedb/src/control/insert_select/orchestrator.rs index e59887a91..a44dd75fa 100644 --- a/nodedb/src/control/insert_select/orchestrator.rs +++ b/nodedb/src/control/insert_select/orchestrator.rs @@ -84,15 +84,6 @@ pub(crate) async fn run_insert_select( database_id: DatabaseId, req: &CopyRequest<'_>, ) -> crate::Result { - // Each page applies on the target's vShard alone, so it folds only - // co-resident balances. - crate::control::planner::materialized_sum::refuse_cross_shard_orchestration( - state, - req.target_collection, - tenant_id, - database_id, - "INSERT ... SELECT", - )?; let spec = resolve_copy_spec( state, tenant_id, diff --git a/nodedb/src/control/merge_orchestrator/orchestrator.rs b/nodedb/src/control/merge_orchestrator/orchestrator.rs index 41149f8d7..4ff1ed673 100644 --- a/nodedb/src/control/merge_orchestrator/orchestrator.rs +++ b/nodedb/src/control/merge_orchestrator/orchestrator.rs @@ -121,15 +121,6 @@ pub async fn run_authorized_merge( /// Returns the `{"affected": N}` (or RETURNING-rows) response the Data-Plane /// merge handler produces, so the dispatch loops render the same command tag. pub(crate) async fn run_merge(state: &SharedState, args: MergeArgs<'_>) -> crate::Result { - // The apply lands on the target's vShard alone, so it folds only - // co-resident balances. - crate::control::planner::materialized_sum::refuse_cross_shard_orchestration( - state, - args.target_collection, - args.tenant_id, - args.database_id, - "MERGE", - )?; let catalog = state.credentials.catalog(); let target_bare = bare_collection_name(args.database_id, args.target_collection); let target = catalog diff --git a/nodedb/src/control/planner/materialized_sum/mod.rs b/nodedb/src/control/planner/materialized_sum/mod.rs index 67b68e5f3..825a2410c 100644 --- a/nodedb/src/control/planner/materialized_sum/mod.rs +++ b/nodedb/src/control/planner/materialized_sum/mod.rs @@ -29,7 +29,7 @@ pub use cross_shard::append_cross_shard_balance_tasks; pub use extract::join_value_from_body; pub use index::MaterializedSumIndex; pub use resolve::{ - drives_cross_shard_sum, refuse_cross_shard_orchestration, resolve_materialized_sum_targets, - resolve_sum_targets_for_bodies, source_drives_bindings, + drives_cross_shard_sum, resolve_materialized_sum_targets, resolve_sum_targets_for_bodies, + source_drives_bindings, }; pub use resolve_target::ResolvedTargets; diff --git a/nodedb/src/control/planner/materialized_sum/predicate.rs b/nodedb/src/control/planner/materialized_sum/predicate.rs index 4b448843c..858847451 100644 --- a/nodedb/src/control/planner/materialized_sum/predicate.rs +++ b/nodedb/src/control/planner/materialized_sum/predicate.rs @@ -58,10 +58,16 @@ struct PredicateScope { /// collection drives a binding — so the caller knows the op is accounted for and /// does not also run the body-driven pass over it. `Ok(None)` means `op` is not /// predicate-driven and the caller still owns it. +/// +/// `txn_id` is the source task's own transaction, which each shipped balance +/// task inherits. `read_txn` is the open transaction the statement runs in: +/// the recon scan reads through its staging overlay, as the staged write +/// itself matches rows. pub(super) async fn resolve_predicate_sum_targets( state: &SharedState, op: &mut DocumentOp, txn_id: Option, + read_txn: Option, tenant_id: TenantId, database_id: DatabaseId, trace_id: TraceId, @@ -79,6 +85,7 @@ pub(super) async fn resolve_predicate_sum_targets( state, tenant_id, database_id, + read_txn, &scope.collection, // Cloned rather than moved: `scope` is still needed below to fold the // images from this same scan, and a filter blob is negligible beside the diff --git a/nodedb/src/control/planner/materialized_sum/recon.rs b/nodedb/src/control/planner/materialized_sum/recon.rs index 4e741e9ca..8fdb0e0fc 100644 --- a/nodedb/src/control/planner/materialized_sum/recon.rs +++ b/nodedb/src/control/planner/materialized_sum/recon.rs @@ -21,6 +21,16 @@ //! vShard returns nothing, which will silently under-resolve and leave the //! write with no target to address. //! +//! # Transaction view +//! +//! Inside a transaction block the read carries the transaction's id. The Data +//! Plane then reads the transaction's staging overlay over committed base, as +//! an in-transaction `SELECT` does. A statement's write applies on top of the +//! rows the transaction's earlier statements staged, so its images are read +//! there too. A base-only read misses a row the transaction inserted and +//! returns the base value of a row it rewrote. The delta settled from such an +//! image is wrong by that row's staged contribution. +//! //! # Plane discipline //! //! Runs on the coordinator's Control Plane (Tokio). The scan goes through the @@ -29,7 +39,7 @@ use nodedb_types::{Surrogate, TenantId}; use crate::control::state::SharedState; -use crate::types::{DatabaseId, Lsn, TraceId}; +use crate::types::{DatabaseId, Lsn, TraceId, TxnId}; use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan}; /// What a plan-time reconnaissance read observed, and the version it observed @@ -63,10 +73,14 @@ pub(crate) struct ReconRead { /// /// Empty `filters` means "no WHERE clause" — every row, which is what `TRUNCATE` /// needs. +/// +/// `txn_id` is the open transaction the statement runs in, `None` outside a +/// transaction block. See the module's transaction view. pub(in crate::control::planner) async fn recon_scan_rows( state: &SharedState, tenant_id: TenantId, database_id: DatabaseId, + txn_id: Option, collection: &str, filters: Vec, ) -> crate::Result>> { @@ -85,7 +99,7 @@ pub(in crate::control::planner) async fn recon_scan_rows( prefilter: None, }); - let read = execute_read(state, tenant_id, database_id, collection, scan_plan).await?; + let read = execute_read(state, tenant_id, database_id, txn_id, collection, scan_plan).await?; let mut rows = Vec::new(); for payload in &read.rows { rows.extend(decode_rows(payload.as_slice())); @@ -111,10 +125,15 @@ pub(in crate::control::planner) async fn recon_scan_rows( /// /// Identity is the surrogate, exactly as on the write path — `document_id` is /// the user-facing primary key and carries no storage addressing. +/// +/// `txn_id` is the open transaction the statement runs in, `None` outside a +/// transaction block. A row this transaction staged reads as staged, and a row +/// it staged a delete of reads as absent. pub(crate) async fn recon_point_row( state: &SharedState, tenant_id: TenantId, database_id: DatabaseId, + txn_id: Option, collection: &str, document_id: &str, surrogate: Surrogate, @@ -133,7 +152,7 @@ pub(crate) async fn recon_point_row( valid_at_ms: None, }); - let read = execute_read(state, tenant_id, database_id, collection, get_plan).await?; + let read = execute_read(state, tenant_id, database_id, txn_id, collection, get_plan).await?; // A point get answers with the row's normalized MessagePack body, and with // an EMPTY payload when the row is absent. Ok(ReconRead { @@ -157,10 +176,16 @@ pub(crate) async fn recon_point_row( /// The gateway notes the node that served each vShard it read. The read /// validates on `collection`'s vShard, so that vShard's note names the node /// whose WAL numbers `read_version_lsn`. +/// +/// `read_version_lsn` is the collection's COMMITTED write floor whether or not +/// `txn_id` is set: a staged write appends no WAL record and moves no write +/// version. So an entry stamped with it still detects a concurrent commit to +/// the rows read, and never this transaction's own staged writes. async fn execute_read( state: &SharedState, tenant_id: TenantId, database_id: DatabaseId, + txn_id: Option, collection: &str, plan: PhysicalPlan, ) -> crate::Result>>> { @@ -173,7 +198,9 @@ async fn execute_read( tenant_id, trace_id: TraceId::ZERO, database_id, - txn_id: None, + // The transaction's staging overlay over committed base, as an + // in-transaction read sees it. + txn_id, linearizable: true, }; // A shard verdict keeps its own typed error. diff --git a/nodedb/src/control/planner/materialized_sum/resolve.rs b/nodedb/src/control/planner/materialized_sum/resolve.rs index 5c48c7a61..e4c7c7ec3 100644 --- a/nodedb/src/control/planner/materialized_sum/resolve.rs +++ b/nodedb/src/control/planner/materialized_sum/resolve.rs @@ -20,7 +20,7 @@ use super::stored::stored_row_scope; use crate::control::server::shared::session::read_set::ReadSetEntry; use crate::control::server::surrogate_exchange::lookup_surrogate_routed; use crate::control::state::SharedState; -use crate::types::{DatabaseId, TenantId, TraceId}; +use crate::types::{DatabaseId, TenantId, TraceId, TxnId}; /// Resolve the materialized-sum target rows for every document write in /// `tasks`, storing the result in that op's `resolved_sum_targets`. @@ -43,11 +43,18 @@ use crate::types::{DatabaseId, TenantId, TraceId}; /// The returned [`ReadSetEntry`]s cover the folded images; the caller must /// union them into the dispatch read-set so Calvin OCC aborts the /// statement on concurrent source writes, before any row moves. +/// +/// `read_txn` is the open transaction the statement runs in, `None` outside a +/// transaction block. Every source read goes through its staging overlay, so +/// each image is the row as this transaction sees it. A planned task's own +/// `txn_id` does not say this: the planner leaves it unset, and the staging +/// gate stamps it only after this pass. pub async fn resolve_materialized_sum_targets( state: &SharedState, tasks: &mut Vec, tenant_id: TenantId, database_id: DatabaseId, + read_txn: Option, trace_id: TraceId, ) -> crate::Result> { let schema_version = state.schema_version.current(); @@ -67,6 +74,7 @@ pub async fn resolve_materialized_sum_targets( state, op, txn_id, + read_txn, tenant_id, database_id, trace_id, @@ -81,7 +89,7 @@ pub async fn resolve_materialized_sum_targets( // end before the resolution is written back through `&mut op`. let outcome = { let carried = value_carrying(op); - let stored = stored_row_scope(op); + let stored = stored_row_scope(op, read_txn); // The two classifications name the same collection when both apply; // an op that is neither carries no join value at all and is skipped. let Some(collection) = stored @@ -286,8 +294,10 @@ fn set_resolved(op: &mut DocumentOp, resolved: Vec) { /// pass, a write pass) that never passes through /// [`resolve_materialized_sum_targets`]. Without this they will ship an empty /// resolution and the Data-Plane fold will have no target to address. Every -/// binding they resolve is co-resident: -/// [`refuse_cross_shard_orchestration`] refuses the others first. +/// binding they resolve is co-resident: a statement that moves a cross-shard +/// sum runs in an implicit transaction on every protocol and never reaches +/// them +/// ([`statement_needs_implicit_txn`](crate::control::server::shared::txn_route::statement_needs_implicit_txn)). /// /// `source_collection` is the db-qualified name as it appears on the plan. /// Returns an empty vec — and issues no lookup at all — when the collection @@ -365,37 +375,6 @@ pub fn drives_cross_shard_sum( })) } -/// Refuse a Control-Plane orchestrated write into `source_collection` when it -/// drives a cross-shard materialized sum. `statement` names the statement -/// kind for the error. -/// -/// An orchestrator applies on the written collection's vShard alone. A -/// cross-shard balance travels on an `ApplyBalanceDelta` task that only a -/// transaction commits with the source rows. So the SQL and native protocols -/// run such a statement in an implicit transaction and never reach the -/// orchestrator with it. A route that reaches it anyway is refused here: -/// folding the balance on the source's core writes a row that vShard does -/// not own. -pub fn refuse_cross_shard_orchestration( - state: &SharedState, - source_collection: &str, - tenant_id: TenantId, - database_id: DatabaseId, - statement: &str, -) -> crate::Result<()> { - if drives_cross_shard_sum(state, source_collection, tenant_id, database_id)? { - return Err(crate::Error::FeatureNotSupported { - detail: format!( - "{statement} into '{source_collection}' moves a materialized sum whose target \ - lives on another vShard, and this route runs it outside a transaction. Run \ - the statement over the SQL or native protocol, which commits the balance \ - with the source rows." - ), - }); - } - Ok(()) -} - /// Resolve one already-extracted join VALUE to its target row's surrogate. /// /// `lookup_surrogate_routed`, never `assign_surrogate_routed`: a join value that @@ -600,7 +579,7 @@ mod tests { .expect("bind target row"); let mut tasks = vec![insert_task("entries", body("acc-1"))]; - resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, TraceId::ZERO) + resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, None, TraceId::ZERO) .await .expect("resolution succeeds"); @@ -651,7 +630,7 @@ mod tests { ); let mut tasks = vec![insert_task("entries", body("acc-1"))]; - resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, TraceId::ZERO) + resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, None, TraceId::ZERO) .await .expect("resolution succeeds"); @@ -685,9 +664,10 @@ mod tests { declare_binding(&state); let mut tasks = vec![insert_task("entries", body("acc-missing"))]; - let error = resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, TraceId::ZERO) - .await - .expect_err("a missing target must fail the statement"); + let error = + resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, None, TraceId::ZERO) + .await + .expect_err("a missing target must fail the statement"); match error { crate::Error::MaterializedSumTargetNotFound { @@ -711,7 +691,7 @@ mod tests { declare_binding(&state); let mut tasks = vec![insert_task("unrelated", body("acc-1"))]; - resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, TraceId::ZERO) + resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, None, TraceId::ZERO) .await .expect("resolution succeeds"); @@ -769,7 +749,7 @@ mod tests { post_set_op: PostSetOp::None, txn_id: None, }]; - resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, TraceId::ZERO) + resolve_materialized_sum_targets(&state, &mut tasks, TENANT, DB, None, TraceId::ZERO) .await .expect("resolution succeeds"); @@ -831,6 +811,7 @@ mod tests { &mut tasks, TENANT, DB, + txn, TraceId::ZERO, ) .await @@ -871,32 +852,21 @@ mod tests { ); } - /// A source that drives a cross-shard target is reported, and the - /// autocommit orchestrators refuse it with the statement and collection - /// named. A collection that drives nothing passes. + /// A source that drives a cross-shard target is reported. A collection + /// that drives nothing is not. #[test] - fn a_cross_shard_source_refuses_orchestration() { + fn a_cross_shard_source_is_reported() { let (state, _directory) = test_state(); declare_binding(&state); assert!(drives_cross_shard_sum(&state, "entries", TENANT, DB).expect("index probe")); assert!(!drives_cross_shard_sum(&state, "unrelated", TENANT, DB).expect("index probe")); - - match refuse_cross_shard_orchestration(&state, "entries", TENANT, DB, "INSERT ... SELECT") { - Err(crate::Error::FeatureNotSupported { detail }) => { - assert!(detail.contains("INSERT ... SELECT"), "{detail}"); - assert!(detail.contains("'entries'"), "{detail}"); - } - other => panic!("expected FeatureNotSupported, got {other:?}"), - } - refuse_cross_shard_orchestration(&state, "unrelated", TENANT, DB, "MERGE") - .expect("a collection with no binding orchestrates"); } /// A source whose every target shares its vShard rides the source write, - /// so it neither reports a cross-shard sum nor refuses orchestration. + /// so it reports no cross-shard sum. #[test] - fn a_co_resident_source_orchestrates() { + fn a_co_resident_source_is_not_cross_shard() { let (state, _directory) = test_state(); let (source, target) = ("uo_entries", "uo_accounts"); assert_eq!( @@ -925,7 +895,5 @@ mod tests { state.materialized_sum_index.invalidate(); assert!(!drives_cross_shard_sum(&state, source, TENANT, DB).expect("index probe")); - refuse_cross_shard_orchestration(&state, source, TENANT, DB, "UPDATE ... FROM") - .expect("a co-resident sum orchestrates"); } } diff --git a/nodedb/src/control/planner/materialized_sum/settle.rs b/nodedb/src/control/planner/materialized_sum/settle.rs index 1f279aef3..8eeffb5bc 100644 --- a/nodedb/src/control/planner/materialized_sum/settle.rs +++ b/nodedb/src/control/planner/materialized_sum/settle.rs @@ -44,11 +44,11 @@ //! `INSERT ... SELECT` run this same pass on the point writes they emit. The //! autocommit orchestrators resolve their rows through //! [`resolve_sum_targets_for_bodies`](super::resolve::resolve_sum_targets_for_bodies) -//! and apply on the source's vShard alone, so they refuse a source with a -//! cross-shard target -//! ([`refuse_cross_shard_orchestration`](super::resolve::refuse_cross_shard_orchestration)). -//! The SQL and native protocols run such a statement through the expanders -//! in an implicit transaction instead. +//! and apply on the source's vShard alone, so they fold co-resident targets +//! only. A source with a cross-shard target never reaches them: every +//! protocol runs such a statement through the expanders in an implicit +//! transaction +//! ([`statement_needs_implicit_txn`](crate::control::server::shared::txn_route::statement_needs_implicit_txn)). //! //! # The images are only as good as the version they were read at //! @@ -276,12 +276,20 @@ fn image_read_entry( }, read_lsn: input.read_version_lsn, read_version_lsn: input.read_version_lsn, - // A DERIVATION read, never a read-your-own-write. The image was read - // from committed base state before this transaction existed, and the - // delta shipped to the target rests entirely on it; the statement writes - // the source collection too, so an entry marked `Session` here will be + // A DERIVATION read, never a read-your-own-write. The delta shipped to + // the target rests entirely on the image. The statement writes the + // source collection too, so an entry marked `Session` here will be // dropped by the own-write exclusion and the fold will never be // validated against a concurrent writer. + // + // Inside a transaction the image is read through its staging overlay. + // `read_version_lsn` is still the committed write floor, because a + // staged write moves no write version. So the entry aborts the commit + // when another transaction commits a write to the row first. It never + // aborts on this transaction's own staged writes, which the image + // already includes. For a row that exists only in the overlay, the + // entry asserts that the base still holds no committed write to that + // surrogate. A concurrent commit of that row invalidates it. origin: ReadOrigin::PlanDerivation, home: None, home_node: input.served_by, diff --git a/nodedb/src/control/planner/materialized_sum/stored.rs b/nodedb/src/control/planner/materialized_sum/stored.rs index 138abac09..0fa8a4125 100644 --- a/nodedb/src/control/planner/materialized_sum/stored.rs +++ b/nodedb/src/control/planner/materialized_sum/stored.rs @@ -24,6 +24,7 @@ use std::sync::Arc; use nodedb_physical::physical_plan::{DocumentOp, MaterializedSumBinding, UpdateValue}; use nodedb_types::Surrogate; +use nodedb_types::id::TxnId; use super::recon::recon_point_row; use super::resolve::lookup_join_value; @@ -69,6 +70,10 @@ pub(super) struct StoredRowScope<'a> { pub updates: &'a [(String, UpdateValue)], /// How this shape's post-image is formed. pub post_image: PostImage<'a>, + /// The open transaction the write runs in, `None` outside a transaction + /// block. The stored row is read through its staging overlay, so the + /// pre-image is the row as this transaction sees it. + pub txn_id: Option, } /// What reading the stored row produced: the write's pre-/post-image pairs @@ -89,7 +94,13 @@ pub(super) struct StoredImages { /// `Upsert` (a join-column rewrite debits the leaving target, named only by /// the stored image). `PointInsert`/`BatchInsert` are absent: rows are new /// by construction, so there's no pre-image to read. -pub(super) fn stored_row_scope(op: &DocumentOp) -> Option> { +/// +/// `txn_id` is the open transaction the write runs in, `None` outside a +/// transaction block. +pub(super) fn stored_row_scope( + op: &DocumentOp, + txn_id: Option, +) -> Option> { match op { // A key unbound in its database names no stored row. DocumentOp::PointUpdate { @@ -104,6 +115,7 @@ pub(super) fn stored_row_scope(op: &DocumentOp) -> Option> { surrogate, updates: updates.as_slice(), post_image: PostImage::Assigned, + txn_id, }), // A delete assigns nothing: the row's pre-image join value is the whole // of what it owes. @@ -118,6 +130,7 @@ pub(super) fn stored_row_scope(op: &DocumentOp) -> Option> { surrogate, updates: &[], post_image: PostImage::Removed, + txn_id, }), // A put replaces the row wholesale, so its post-image is the submitted // body — already resolved from the body — and the stored row is needed @@ -134,6 +147,7 @@ pub(super) fn stored_row_scope(op: &DocumentOp) -> Option> { surrogate: *surrogate, updates: &[], post_image: PostImage::Body(value.as_slice()), + txn_id, }), // On the conflict branch an upsert's post-image is the stored row with // `on_conflict_updates` applied, so those assignments decide the target @@ -152,6 +166,7 @@ pub(super) fn stored_row_scope(op: &DocumentOp) -> Option> { surrogate: *surrogate, updates: on_conflict_updates.as_slice(), post_image: PostImage::BodyOrAssigned(value.as_slice()), + txn_id, }), DocumentOp::PointInsert { .. } | DocumentOp::BatchInsert { .. } @@ -201,6 +216,7 @@ pub(super) async fn extend_with_stored_row( state, tenant_id, database_id, + scope.txn_id, scope.collection, scope.document_id, scope.surrogate, @@ -395,7 +411,7 @@ mod tests { #[test] fn every_point_shape_names_the_stored_row_it_rewrites() { for op in [point_delete(), point_update(), point_put(), upsert()] { - let scope = stored_row_scope(&op) + let scope = stored_row_scope(&op, None) .unwrap_or_else(|| panic!("a point write must name its stored row: {op:?}")); assert_eq!(scope.collection, "entries"); assert_eq!(scope.document_id, "e1"); @@ -410,7 +426,7 @@ mod tests { /// read and no round trip to spend learning that. #[test] fn an_insert_names_no_stored_row() { - assert!(stored_row_scope(&point_insert()).is_none()); + assert!(stored_row_scope(&point_insert(), None).is_none()); } /// The assignments travel on the scope, so a write that rewrites the join @@ -420,7 +436,7 @@ mod tests { #[test] fn assignments_that_rewrite_the_join_column_travel_on_the_scope() { for op in [point_update(), upsert()] { - let scope = stored_row_scope(&op).expect("a point write names its stored row"); + let scope = stored_row_scope(&op, None).expect("a point write names its stored row"); assert_eq!( scope.updates.len(), 1, @@ -435,7 +451,7 @@ mod tests { #[test] fn the_shapes_without_assignments_carry_none() { for op in [point_delete(), point_put()] { - let scope = stored_row_scope(&op).expect("a point write names its stored row"); + let scope = stored_row_scope(&op, None).expect("a point write names its stored row"); assert!(scope.updates.is_empty(), "{op:?}"); } } @@ -486,7 +502,7 @@ mod tests { body("acc-1", 60), vec![("amount".to_string(), excluded("amount"))], ); - let scope = stored_row_scope(&op).expect("an upsert names its stored row"); + let scope = stored_row_scope(&op, None).expect("an upsert names its stored row"); let stored = serde_json::json!({"account_id": "acc-1", "amount": 25}); let images = images_of(&scope, Some(&stored)).expect("images"); @@ -501,7 +517,7 @@ mod tests { #[test] fn an_upsert_without_conflict_assignments_merges_the_body() { let op = upsert_with(body("acc-1", 60), Vec::new()); - let scope = stored_row_scope(&op).expect("an upsert names its stored row"); + let scope = stored_row_scope(&op, None).expect("an upsert names its stored row"); let stored = serde_json::json!({"account_id": "acc-1", "amount": 25, "memo": "kept"}); let images = images_of(&scope, Some(&stored)).expect("images"); @@ -529,7 +545,7 @@ mod tests { ("amount".to_string(), excluded("amount")), ], ); - let scope = stored_row_scope(&op).expect("an upsert names its stored row"); + let scope = stored_row_scope(&op, None).expect("an upsert names its stored row"); let stored = serde_json::json!({"account_id": "acc-1", "amount": 25}); let images = images_of(&scope, Some(&stored)).expect("images"); @@ -552,7 +568,7 @@ mod tests { body("acc-1", 60), vec![("amount".to_string(), excluded("amount"))], ); - let scope = stored_row_scope(&op).expect("an upsert names its stored row"); + let scope = stored_row_scope(&op, None).expect("an upsert names its stored row"); let images = images_of(&scope, None).expect("images"); assert_eq!(images.len(), 1); @@ -575,7 +591,7 @@ mod tests { rls_filters: Vec::new(), resolved_sum_targets: Vec::new(), }; - let scope = stored_row_scope(&op).expect("a put names its stored row"); + let scope = stored_row_scope(&op, None).expect("a put names its stored row"); let stored = serde_json::json!({"account_id": "acc-1", "amount": 25}); let images = images_of(&scope, Some(&stored)).expect("images"); diff --git a/nodedb/src/control/planner/period_lock/lookup.rs b/nodedb/src/control/planner/period_lock/lookup.rs index e9d207c24..205c26c6a 100644 --- a/nodedb/src/control/planner/period_lock/lookup.rs +++ b/nodedb/src/control/planner/period_lock/lookup.rs @@ -4,7 +4,7 @@ use crate::control::server::surrogate_exchange::lookup_surrogate_routed; use crate::control::state::SharedState; -use crate::types::{DatabaseId, TenantId, TraceId}; +use crate::types::{DatabaseId, TenantId, TraceId, TxnId}; /// Request scope shared by every per-row period-lock resolution call within /// one [`resolve_period_lock_targets`](super::resolve::resolve_period_lock_targets) @@ -14,6 +14,11 @@ pub(super) struct PeriodLockScope<'a> { pub state: &'a SharedState, pub tenant_id: TenantId, pub database_id: DatabaseId, + /// The open transaction the statement runs in, `None` outside a + /// transaction block. Every stored-row read goes through its staging + /// overlay, so the period a write gates on is the row as this transaction + /// sees it. + pub read_txn: Option, pub trace_id: TraceId, } diff --git a/nodedb/src/control/planner/period_lock/point_update.rs b/nodedb/src/control/planner/period_lock/point_update.rs index e64fb8c1f..7cbb8245c 100644 --- a/nodedb/src/control/planner/period_lock/point_update.rs +++ b/nodedb/src/control/planner/period_lock/point_update.rs @@ -29,6 +29,7 @@ pub(super) async fn resolve_update_period_values( scope.state, scope.tenant_id, scope.database_id, + scope.read_txn, collection, document_id, surrogate, diff --git a/nodedb/src/control/planner/period_lock/predicate.rs b/nodedb/src/control/planner/period_lock/predicate.rs index ccf79a68d..2855b9bdf 100644 --- a/nodedb/src/control/planner/period_lock/predicate.rs +++ b/nodedb/src/control/planner/period_lock/predicate.rs @@ -40,6 +40,7 @@ pub(super) async fn resolve_predicate_period_values( scope.state, scope.tenant_id, scope.database_id, + scope.read_txn, collection, filters, ) diff --git a/nodedb/src/control/planner/period_lock/resolve.rs b/nodedb/src/control/planner/period_lock/resolve.rs index bcc7f2856..d63f3b6ca 100644 --- a/nodedb/src/control/planner/period_lock/resolve.rs +++ b/nodedb/src/control/planner/period_lock/resolve.rs @@ -53,7 +53,7 @@ use super::point_update::resolve_update_period_values; use super::predicate::{PeriodLockEffect, resolve_predicate_period_values}; use super::singular::{resolve_batch_period_values, singular_period_value}; use crate::control::state::SharedState; -use crate::types::{DatabaseId, TenantId, TraceId}; +use crate::types::{DatabaseId, TenantId, TraceId, TxnId}; /// Resolve the period-lock reference row for every write in `tasks` that a /// period lock gates, appending the entry to the op's `resolved_sum_targets` @@ -63,11 +63,15 @@ use crate::types::{DatabaseId, TenantId, TraceId}; /// value) adds no entry: `check_period_lock` treats an absent entry as an /// unknown period and refuses the write, exactly as it treats a genuinely /// closed one. +/// +/// `read_txn` is the open transaction the statement runs in, `None` outside a +/// transaction block. A stored row is read through its staging overlay. pub async fn resolve_period_lock_targets( state: &SharedState, tasks: &mut [PhysicalTask], tenant_id: TenantId, database_id: DatabaseId, + read_txn: Option, trace_id: TraceId, ) -> crate::Result<()> { let catalog = state.credentials.catalog(); @@ -75,6 +79,7 @@ pub async fn resolve_period_lock_targets( state, tenant_id, database_id, + read_txn, trace_id, }; for task in tasks.iter_mut() { @@ -180,9 +185,7 @@ pub async fn resolve_period_lock_targets( continue; } - let Some(period_key) = - singular_period_value(state, op, &collection, &def, tenant_id, database_id).await? - else { + let Some(period_key) = singular_period_value(&scope, op, &collection, &def).await? else { continue; }; diff --git a/nodedb/src/control/planner/period_lock/singular.rs b/nodedb/src/control/planner/period_lock/singular.rs index d75baa496..b6e635321 100644 --- a/nodedb/src/control/planner/period_lock/singular.rs +++ b/nodedb/src/control/planner/period_lock/singular.rs @@ -5,7 +5,7 @@ use nodedb_physical::physical_plan::{DocumentOp, ResolvedSumTarget}; -use super::lookup::lookup_period_surrogate; +use super::lookup::{PeriodLockScope, lookup_period_surrogate}; use crate::control::planner::materialized_sum::recon::recon_point_row; use crate::control::planner::materialized_sum::{ResolvedTargets, join_value_from_body}; use crate::control::security::catalog::PeriodLockDef; @@ -21,12 +21,10 @@ use crate::types::{DatabaseId, TenantId, TraceId}; /// off its submitted body. `PointDelete` carries no body, so its value comes /// off the row it is about to remove. pub(super) async fn singular_period_value( - state: &SharedState, + scope: &PeriodLockScope<'_>, op: &DocumentOp, collection: &str, def: &PeriodLockDef, - tenant_id: TenantId, - database_id: DatabaseId, ) -> crate::Result> { match op { DocumentOp::PointPut { value, .. } @@ -42,9 +40,10 @@ pub(super) async fn singular_period_value( return Ok(None); }; let read = recon_point_row( - state, - tenant_id, - database_id, + scope.state, + scope.tenant_id, + scope.database_id, + scope.read_txn, collection, document_id, *surrogate, diff --git a/nodedb/src/control/planner/procedural/executor/core/dispatch.rs b/nodedb/src/control/planner/procedural/executor/core/dispatch.rs index 23b086141..72cb603b6 100644 --- a/nodedb/src/control/planner/procedural/executor/core/dispatch.rs +++ b/nodedb/src/control/planner/procedural/executor/core/dispatch.rs @@ -129,6 +129,9 @@ impl<'a> StatementExecutor<'a> { // locks) join the statement as they do for a client statement. The // plan never carries a Calvin OLLP prediction or a resolved write: // the planner emits neither, so every task takes the staging gate. + // They read their source rows through the transaction this statement + // stages into, so they see its earlier statements. + let read_txn = self.read_txn().await?; let (tasks, planned, lease_scope, sum_target_reads) = crate::control::server::shared::retry::retry_on_schema_change( &self.state.lease_drain, @@ -149,6 +152,7 @@ impl<'a> StatementExecutor<'a> { &mut tasks, self.tenant_id, self.database_id, + read_txn, crate::types::TraceId::ZERO, ) .await?; @@ -199,6 +203,20 @@ impl<'a> StatementExecutor<'a> { .map_err(crate::Error::from) } + /// The id of the transaction the next statement stages into: the open + /// one, else the joined statement's. `None` when neither exists yet, so + /// the transaction the statement opens has staged nothing. + async fn read_txn(&self) -> crate::Result> { + let txn = self.txn.lock().await; + if let Some(open) = txn.as_ref() { + let ctx = open.txn_ctx()?; + return Ok(ctx.sessions.tx_id(ctx.session_id)); + } + Ok(self + .joined + .and_then(|ctx| ctx.sessions.tx_id(ctx.session_id))) + } + /// The open transaction, begun now when none is open. A joined body /// joins its statement's transaction instead. async fn open_txn<'g>( diff --git a/nodedb/src/control/server/http/routes/atomic_statement.rs b/nodedb/src/control/server/http/routes/atomic_statement.rs new file mode 100644 index 000000000..ca75cb211 --- /dev/null +++ b/nodedb/src/control/server/http/routes/atomic_statement.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The shared atomic routes for the HTTP query routes: `/v1/query`, +//! `/v1/query/stream` and WebSocket RPC. +//! +//! Each route calls [`route_http_statement`] after admission. A statement +//! the shared router takes runs there, through the same implicit +//! transaction, implicit-edge gate and Calvin commit native and pgwire use, +//! and answers as JSON rows. Every other statement comes back for the +//! route's own per-task loop. + +use crate::control::server::response_shape::cell::row_to_wire_json; +use crate::control::server::response_shape::types::FoldedTag; +use crate::control::server::shared::session::DetachedTxnScope; +use crate::control::server::shared::statement_exec::{ + AtomicRoute, AtomicStatement, StatementAnswer, StatementExec, calvin_answer, + route_atomic_statement, +}; + +/// What the shared router did with a statement. +pub(super) enum RoutedStatement { + /// An atomic route ran the statement. Its answer as JSON rows. + Answered(Vec), + /// No atomic route applies. The route runs its per-task loop. + PerTask(AtomicStatement), +} + +/// Route `statement` through the shared atomic routes. +/// +/// These routes carry no session: no transaction block is open, and the +/// cross-shard mode is the default, `Strict`. An implicit transaction runs +/// on a private session of its own. +pub(super) async fn route_http_statement( + exec: &StatementExec<'_>, + statement: AtomicStatement, +) -> crate::Result { + let detached = DetachedTxnScope::new(); + let session = detached.ctx(); + let answer = match route_atomic_statement(exec, statement, &session).await { + Ok(AtomicRoute::Implicit(answer)) => Ok(answer), + Ok(AtomicRoute::Calvin(applied)) => calvin_answer(exec, applied), + Ok(AtomicRoute::PerTask(statement)) => return Ok(RoutedStatement::PerTask(statement)), + Err(error) => Err(error), + }; + let answer = answer.map_err(|error| error.into_error(exec.tenant_id()))?; + Ok(RoutedStatement::Answered(answer_json_rows(answer))) +} + +/// A statement's answer as JSON rows, each keyed by its cell keys as +/// `shape_http_payload` keys a row. A write that answers no rows answers one +/// `{"affected": n}` row: the shape a single write's count takes on these +/// routes. A derived task (a balance move, an implicit edge) adds no count. +fn answer_json_rows(answer: StatementAnswer) -> Vec { + let mut rows: Vec = answer + .rows + .iter() + .flat_map(|shaped| { + shaped + .rows + .iter() + .map(|row| serde_json::Value::Object(row_to_wire_json(row))) + }) + .collect(); + if rows.is_empty() + && let Some(FoldedTag::Dml(outcome)) = answer.tag + { + rows.push(serde_json::json!({ "affected": outcome.affected })); + } + rows +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::server::response_shape::types::DmlOutcome; + + fn answer(tag: Option) -> StatementAnswer { + StatementAnswer { + rows: Vec::new(), + warnings: Vec::new(), + tag, + last_lsn: 0, + } + } + + #[test] + fn a_counted_write_answers_one_affected_row() { + let rows = answer_json_rows(answer(Some(FoldedTag::Dml(DmlOutcome { + verb: "INSERT", + affected: 2, + })))); + assert_eq!(rows, vec![serde_json::json!({ "affected": 2 })]); + } + + #[test] + fn an_opaque_statement_answers_no_rows() { + assert!(answer_json_rows(answer(Some(FoldedTag::Opaque))).is_empty()); + assert!(answer_json_rows(answer(None)).is_empty()); + } +} diff --git a/nodedb/src/control/server/http/routes/mod.rs b/nodedb/src/control/server/http/routes/mod.rs index 43ba25de3..f9e5ae0db 100644 --- a/nodedb/src/control/server/http/routes/mod.rs +++ b/nodedb/src/control/server/http/routes/mod.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 +mod atomic_statement; pub mod auth_key; pub mod auth_session; pub mod cdc; diff --git a/nodedb/src/control/server/http/routes/query/materialized/append.rs b/nodedb/src/control/server/http/routes/query/materialized/append.rs index a3e6b55a8..71e2b86cd 100644 --- a/nodedb/src/control/server/http/routes/query/materialized/append.rs +++ b/nodedb/src/control/server/http/routes/query/materialized/append.rs @@ -1,16 +1,9 @@ // SPDX-License-Identifier: BUSL-1.1 //! Per-task helpers the dispatch loop and the orchestrated plans share: -//! authorization without the clone-write check, metering, and shaping one -//! task's answer into the statement's JSON rows. - -use std::sync::Arc; - -use nodedb_physical::physical_task::PhysicalTask; +//! metering, and shaping one task's answer into the statement's JSON rows. use crate::bridge::envelope::Status; -use crate::control::security::audit::ArcAuditEmitter; -use crate::control::security::identity::AuthenticatedIdentity; use crate::control::security::request_scope::RequestAuthScope; use crate::control::server::response_shape::redaction::QueryRedaction; use crate::control::server::response_shape::request::MaterializedShapeRequest; @@ -25,31 +18,6 @@ use super::super::super::result_shape::{ }; use super::encode::response_error; -/// Authorize one task with no clone-write check. Used only for the -/// Control-Plane orchestrated plans (see `orchestrated`), which are never -/// clone-write shapes. -pub(super) fn authorize_materialized_task( - shared: &crate::control::state::SharedState, - identity: &AuthenticatedIdentity, - task: &PhysicalTask, -) -> crate::Result { - let emitter = ArcAuditEmitter(Arc::clone(&shared.audit)); - crate::control::server::shared::authorization::authorize_task_set( - identity, - std::slice::from_ref(task), - &shared.permissions, - &shared.roles, - &emitter, - ) - .map_err(crate::Error::from)? - .into_tasks() - .into_iter() - .next() - .ok_or_else(|| crate::Error::Internal { - detail: "authorization returned an empty capability set".into(), - }) -} - /// Meter one task's dispatch after its rows are appended to `result_rows` — /// the row count is the delta since `rows_before`. pub(super) fn meter_task_dispatch( diff --git a/nodedb/src/control/server/http/routes/query/materialized/mod.rs b/nodedb/src/control/server/http/routes/query/materialized/mod.rs index b16412010..1f5693dfa 100644 --- a/nodedb/src/control/server/http/routes/query/materialized/mod.rs +++ b/nodedb/src/control/server/http/routes/query/materialized/mod.rs @@ -2,12 +2,13 @@ //! `/v1/query`: materialized (buffer-then-respond) SQL execution. //! -//! - `request.rs`: the entry point through admission. +//! - `request.rs`: the entry point through admission and the shared atomic +//! routes. //! - `shape.rs`: the per-task dispatch loop. //! - `orchestrated.rs`: the plans that run on the Control Plane instead of //! the gateway route, cluster array ops among them. //! - `append.rs`: shaping each task's answer into JSON rows, and the -//! per-task authorization and metering helpers. +//! per-task metering helper. //! - `encode.rs`: maps errors onto the HTTP surface. mod append; diff --git a/nodedb/src/control/server/http/routes/query/materialized/orchestrated.rs b/nodedb/src/control/server/http/routes/query/materialized/orchestrated.rs index 3cb1ef0c8..50d6705ea 100644 --- a/nodedb/src/control/server/http/routes/query/materialized/orchestrated.rs +++ b/nodedb/src/control/server/http/routes/query/materialized/orchestrated.rs @@ -19,10 +19,10 @@ use nodedb_physical::physical_task::PhysicalTask; use crate::bridge::envelope::{PhysicalPlan, Response, Status}; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::shared::cluster_array_dispatch::{is_cluster_array, run_cluster_array}; +use crate::control::server::shared::statement_exec::authorize_one_task; use crate::control::state::SharedState; use super::super::super::super::auth::ApiError; -use super::append::authorize_materialized_task; use super::encode::{gateway_error, response_error}; /// The payload an orchestrated plan answered with. @@ -39,7 +39,7 @@ pub(super) async fn run_orchestrated( identity: &AuthenticatedIdentity, task: &PhysicalTask, ) -> Result, ApiError> { - let authorize = || authorize_materialized_task(shared, identity, task).map_err(gateway_error); + let authorize = || authorize_one_task(shared, identity, task).map_err(gateway_error); if crate::control::array_catalog::ddl::is_array_ddl(&task.plan) { let response = diff --git a/nodedb/src/control/server/http/routes/query/materialized/request.rs b/nodedb/src/control/server/http/routes/query/materialized/request.rs index 99ff58e79..65bb7af40 100644 --- a/nodedb/src/control/server/http/routes/query/materialized/request.rs +++ b/nodedb/src/control/server/http/routes/query/materialized/request.rs @@ -7,10 +7,12 @@ use axum::http::HeaderMap; use axum::response::IntoResponse; use crate::control::security::audit::ArcAuditEmitter; +use crate::control::sequence::SessionSequenceAccess; use crate::control::server::shared::authorization::authorize_database; use crate::control::server::shared::plan_admission::{ PlanAdmissionRequest, plan_authorize_and_admit, }; +use crate::control::server::shared::statement_exec::{AtomicStatement, StatementExec}; use super::super::super::super::auth::{ ApiError, AppState, build_request_scope, resolve_auth_parts, @@ -18,6 +20,7 @@ use super::super::super::super::auth::{ use super::super::super::super::peer::PeerAddr; use super::super::super::super::transport::ClientTransport; use super::super::super::super::types::{HttpQueryRequest, HttpQueryResponse}; +use super::super::super::atomic_statement::{RoutedStatement, route_http_statement}; use super::super::super::result_shape::ddl_results_to_json; use super::super::{DatabaseQueryParam, resolve_database_id}; use super::encode::ddl_error_to_api; @@ -136,6 +139,40 @@ pub async fn query( // A statement admitted under a lease this node then loses ends with a // retryable error: a read mid-flight, a write only before dispatch. lease_scope.check_not_revoked().map_err(ApiError::from)?; + + // The shared atomic routes run a statement whose tasks commit together: + // the implicit transaction, the implicit-edge gate, and a Calvin commit + // for a write that spans vShards. + // HTTP carries no session: `nextval` advances the registry, `currval` + // reports "not yet called in this session". + let sequences = SessionSequenceAccess::for_session(&state.shared, None, database_id, tenant_id); + let exec = StatementExec { + state: &state.shared, + identity: &identity, + scope: &scope, + output_schema: Some(&output_schema), + database_id, + sequences: &sequences, + }; + let routed = route_http_statement( + &exec, + AtomicStatement { + tasks, + lease_scope, + sum_target_reads: admission.sum_target_reads, + }, + ) + .await + .map_err(ApiError::from)?; + let AtomicStatement { + tasks, lease_scope, .. + } = match routed { + RoutedStatement::Answered(rows) => { + return Ok((rate_limit_headers, axum::Json(HttpQueryResponse::ok(rows)))); + } + RoutedStatement::PerTask(statement) => statement, + }; + let read_only = tasks .iter() .all(|task| !crate::control::server::shared::write_admission::plan_is_write(&task.plan)); diff --git a/nodedb/src/control/server/http/routes/query/ndjson.rs b/nodedb/src/control/server/http/routes/query/ndjson.rs index 4480375ca..0b0708b35 100644 --- a/nodedb/src/control/server/http/routes/query/ndjson.rs +++ b/nodedb/src/control/server/http/routes/query/ndjson.rs @@ -9,6 +9,7 @@ use axum::response::IntoResponse; use crate::control::gateway::GatewayErrorMap; use crate::control::gateway::core::QueryContext; use crate::control::security::audit::ArcAuditEmitter; +use crate::control::sequence::SessionSequenceAccess; use crate::control::server::response_shape::redaction::QueryRedaction; use crate::control::server::response_shape::request::MaterializedShapeRequest; use crate::control::server::response_shape::types::describe_plan; @@ -21,10 +22,14 @@ use crate::control::server::shared::plan_admission::{ PlanAdmissionRequest, plan_authorize_and_admit, }; use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; +use crate::control::server::shared::statement_exec::{ + AtomicStatement, StatementExec, authorize_one_task, +}; use super::super::super::auth::{ApiError, AppState, build_request_scope, resolve_auth_parts}; use super::super::super::peer::PeerAddr; use super::super::super::transport::ClientTransport; +use super::super::atomic_statement::{RoutedStatement, route_http_statement}; use super::super::query_stream::{NdjsonBody, ndjson_body_stream, try_open_stream}; use super::super::result_shape::{HttpShaped, passthrough_to_ndjson, shape_http_payload}; use super::{DatabaseQueryParam, resolve_database_id}; @@ -119,14 +124,55 @@ pub async fn query_ndjson( Ok(admission) => admission, Err(error) => return ApiError::from(error).into_response(), }; - let tasks = admission.tasks; let output_schema = admission.output_schema; - let mut lease_scope = Some(admission.lease_scope); let trace_id = crate::control::trace_context::generate_trace_id(); let _request = state.shared.tenant_request_guard(tenant_id); + // The shared atomic routes run a statement whose tasks commit together: + // the implicit transaction, the implicit-edge gate, and a Calvin commit + // for a write that spans vShards. HTTP carries no session: `nextval` + // advances the registry, `currval` reports "not yet called in this + // session". + let sequences = SessionSequenceAccess::for_session(&state.shared, None, database_id, tenant_id); + let exec = StatementExec { + state: &state.shared, + identity: &identity, + scope: &scope, + output_schema: Some(&output_schema), + database_id, + sequences: &sequences, + }; + let routed = route_http_statement( + &exec, + AtomicStatement { + tasks: admission.tasks, + lease_scope: admission.lease_scope, + sum_target_reads: admission.sum_target_reads, + }, + ) + .await; + let (tasks, lease_scope) = match routed { + Ok(RoutedStatement::PerTask(statement)) => (statement.tasks, statement.lease_scope), + Ok(RoutedStatement::Answered(rows)) => { + let mut ndjson = String::new(); + for row in rows { + ndjson.push_str(&row.to_string()); + ndjson.push('\n'); + } + return ndjson_response(ndjson, rate_limit_headers); + } + // A statement that stopped answers one error line, as a task that + // stopped in the loop below does. + Err(error) => { + let (_status, message) = GatewayErrorMap::to_http(&error); + let line = format!("{}\n", serde_json::json!({ "error": message })); + return ndjson_response(line, rate_limit_headers); + } + }; + let mut lease_scope = Some(lease_scope); + // `Body::from_stream` polls the data-plane stream under normal HTTP backpressure // while its captured lease scope stays alive until body completion or disconnect. match try_open_stream( @@ -225,7 +271,7 @@ pub async fn query_ndjson( let dispatch = async { let result: crate::Result>> = if crate::control::array_catalog::ddl::is_array_ddl(&task.plan) { - match authorize_ndjson_task(&state.shared, &identity, &task) { + match authorize_one_task(&state.shared, &identity, &task) { Ok(authorized_task) => { crate::control::array_catalog::ddl::run_authorized_array_ddl( &state.shared, @@ -239,7 +285,7 @@ pub async fn query_ndjson( } else if is_cluster_array(&task.plan) { // This node's array coordinator routes the op to the // shards that own its cells. - match authorize_ndjson_task(&state.shared, &identity, &task) { + match authorize_one_task(&state.shared, &identity, &task) { Ok(authorized_task) => run_cluster_array(&state.shared, authorized_task) .await .map(|payload| vec![payload]), @@ -251,7 +297,7 @@ pub async fn query_ndjson( nodedb_physical::physical_plan::DocumentOp::InsertSelect { .. } ) ) { - match authorize_ndjson_task(&state.shared, &identity, &task) { + match authorize_one_task(&state.shared, &identity, &task) { Ok(authorized_task) => { crate::control::insert_select::run_authorized_insert_select( &state.shared, @@ -271,7 +317,7 @@ pub async fn query_ndjson( } ) ) { - match authorize_ndjson_task(&state.shared, &identity, &task) { + match authorize_one_task(&state.shared, &identity, &task) { Ok(authorized_task) => { crate::control::merge_orchestrator::run_authorized_merge( &state.shared, @@ -291,7 +337,7 @@ pub async fn query_ndjson( } ) ) { - match authorize_ndjson_task(&state.shared, &identity, &task) { + match authorize_one_task(&state.shared, &identity, &task) { Ok(authorized_task) => { crate::control::update_from_join_orchestrator::run_authorized_update_from_join( &state.shared, @@ -307,7 +353,7 @@ pub async fn query_ndjson( { // A governed columnar predicate UPDATE/DELETE resolves to a concrete row set // before proposing. - match authorize_ndjson_task(&state.shared, &identity, &task) { + match authorize_one_task(&state.shared, &identity, &task) { Ok(authorized_task) => { crate::control::write_resolve::run_authorized_write_resolve( &state.shared, @@ -425,36 +471,18 @@ pub async fn query_ndjson( } } - let mut response = Response::builder() + ndjson_response(ndjson, rate_limit_headers) +} + +/// A materialized NDJSON body with the request's rate-limit headers. +fn ndjson_response( + ndjson: String, + rate_limit_headers: axum::http::HeaderMap, +) -> axum::response::Response { + let mut response = axum::response::Response::builder() .header("Content-Type", "application/x-ndjson") .body(axum::body::Body::from(ndjson)) .unwrap_or_else(|_| (StatusCode::INTERNAL_SERVER_ERROR, "encoding error").into_response()); response.headers_mut().extend(rate_limit_headers); response } - -/// Authorize one task with no clone-write check — used only by the -/// Control-Plane orchestrator branches ahead of the general dispatch tail, -/// whose plan shapes (`InsertSelect`, `Merge`, `UpdateFromJoin`, a governed -/// predicate resolution) are never clone-write shapes. -fn authorize_ndjson_task( - shared: &crate::control::state::SharedState, - identity: &crate::control::security::identity::AuthenticatedIdentity, - task: &nodedb_physical::physical_task::PhysicalTask, -) -> crate::Result { - let emitter = ArcAuditEmitter(Arc::clone(&shared.audit)); - crate::control::server::shared::authorization::authorize_task_set( - identity, - std::slice::from_ref(task), - &shared.permissions, - &shared.roles, - &emitter, - ) - .map_err(crate::Error::from)? - .into_tasks() - .into_iter() - .next() - .ok_or_else(|| crate::Error::Internal { - detail: "authorization returned an empty capability set".into(), - }) -} diff --git a/nodedb/src/control/server/http/routes/ws_rpc/execute_sql.rs b/nodedb/src/control/server/http/routes/ws_rpc/execute_sql.rs index 57611f31c..aa7e286eb 100644 --- a/nodedb/src/control/server/http/routes/ws_rpc/execute_sql.rs +++ b/nodedb/src/control/server/http/routes/ws_rpc/execute_sql.rs @@ -8,6 +8,7 @@ use crate::control::gateway::core::QueryContext; use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::security::request_scope::RequestAuthScope; +use crate::control::sequence::SessionSequenceAccess; use crate::control::server::response_shape::redaction::{QueryRedaction, redact_decoded_value}; use crate::control::server::shared::authorization::authorize_database; use crate::control::server::shared::cluster_array_dispatch::{is_cluster_array, run_cluster_array}; @@ -16,11 +17,17 @@ use crate::control::server::shared::plan_admission::{ PlanAdmissionRequest, plan_authorize_and_admit, }; use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; +use crate::control::server::shared::statement_exec::{ + AtomicStatement, StatementExec, authorize_one_task, +}; use crate::control::state::SharedState; use crate::types::{DatabaseId, TraceId}; +use super::super::atomic_statement::{RoutedStatement, route_http_statement}; + /// Execute SQL and return result as JSON. /// +/// A statement whose tasks commit together takes the shared atomic routes. /// A Control-Plane orchestrated task (array DDL, a cluster array op, /// `INSERT ... SELECT`) runs here. Every other task routes through the /// gateway. @@ -64,12 +71,45 @@ pub async fn execute_sql( trace_id, }) .await?; - let tasks = admission.tasks; - let lease_scope = admission.lease_scope; + let output_schema = admission.output_schema; // A statement admitted under a lease this node then loses ends with a // retryable error: a read mid-flight, a write only before dispatch. - lease_scope.check_not_revoked()?; + admission.lease_scope.check_not_revoked()?; + + // The shared atomic routes run a statement whose tasks commit together: + // the implicit transaction, the implicit-edge gate, and a Calvin commit + // for a write that spans vShards. WebSocket RPC carries no session: + // `nextval` advances the registry, `currval` reports "not yet called in + // this session". + let sequences = SessionSequenceAccess::for_session(shared, None, database_id, tenant_id); + let exec = StatementExec { + state: shared, + identity, + scope: &scope, + output_schema: Some(&output_schema), + database_id, + sequences: &sequences, + }; + let routed = { + let _request = shared.tenant_request_guard(tenant_id); + route_http_statement( + &exec, + AtomicStatement { + tasks: admission.tasks, + lease_scope: admission.lease_scope, + sum_target_reads: admission.sum_target_reads, + }, + ) + .await? + }; + let AtomicStatement { + tasks, lease_scope, .. + } = match routed { + RoutedStatement::Answered(rows) => return Ok(rows_result(rows)), + RoutedStatement::PerTask(statement) => statement, + }; + let read_only = tasks .iter() .all(|task| !crate::control::server::shared::write_admission::plan_is_write(&task.plan)); @@ -99,7 +139,7 @@ pub async fn execute_sql( // Array DDL proposes a replicated catalog entry, never a core task. if crate::control::array_catalog::ddl::is_array_ddl(&task.plan) { - let authorized_task = authorize_ws_rpc_task(shared, identity, &task)?; + let authorized_task = authorize_one_task(shared, identity, &task)?; let resp = crate::control::array_catalog::ddl::run_authorized_array_ddl( shared, authorized_task, @@ -117,7 +157,7 @@ pub async fn execute_sql( // A cluster array op runs through this node's array coordinator, // which routes it to the shards that own its cells. if is_cluster_array(&task.plan) { - let authorized_task = authorize_ws_rpc_task(shared, identity, &task)?; + let authorized_task = authorize_one_task(shared, identity, &task)?; let payload = run_cluster_array(shared, authorized_task).await?; if !payload.is_empty() { let json = @@ -140,7 +180,7 @@ pub async fn execute_sql( nodedb_physical::physical_plan::DocumentOp::InsertSelect { .. }, ) = &task.plan { - let authorized_task = authorize_ws_rpc_task(shared, identity, &task)?; + let authorized_task = authorize_one_task(shared, identity, &task)?; match crate::control::insert_select::run_authorized_insert_select( shared, authorized_task, @@ -194,7 +234,7 @@ pub async fn execute_sql( }, ) = &task.plan { - let authorized_task = authorize_ws_rpc_task(shared, identity, &task)?; + let authorized_task = authorize_one_task(shared, identity, &task)?; match crate::control::merge_orchestrator::run_authorized_merge( shared, authorized_task, @@ -247,7 +287,7 @@ pub async fn execute_sql( }, ) = &task.plan { - let authorized_task = authorize_ws_rpc_task(shared, identity, &task)?; + let authorized_task = authorize_one_task(shared, identity, &task)?; match crate::control::update_from_join_orchestrator::run_authorized_update_from_join( shared, authorized_task, @@ -277,7 +317,7 @@ pub async fn execute_sql( // A governed columnar predicate UPDATE/DELETE resolves to a concrete row set // before proposing. if let Some(resolver) = crate::control::write_resolve::resolver_for_plan(&task.plan) { - let authorized_task = authorize_ws_rpc_task(shared, identity, &task)?; + let authorized_task = authorize_one_task(shared, identity, &task)?; match crate::control::write_resolve::run_authorized_write_resolve( shared, authorized_task, @@ -399,15 +439,7 @@ pub async fn execute_sql( meter_task(shared, &scope, &plan_metering_info, rows_before, &results); } - let outcome: crate::Result = match results.len() { - 0 => Ok(serde_json::Value::Null), - 1 => Ok(results - .into_iter() - .next() - .unwrap_or(serde_json::Value::Null)), - _ => Ok(serde_json::Value::Array(results)), - }; - outcome + Ok(rows_result(results)) }; if read_only { lease_scope.guard(body).await? @@ -416,30 +448,14 @@ pub async fn execute_sql( } } -/// Authorize one task with no clone-write check — used only by the -/// Control-Plane orchestrator branches ahead of the general dispatch tail, -/// whose plan shapes (`InsertSelect`, `Merge`, `UpdateFromJoin`, a governed -/// predicate resolution) are never clone-write shapes. -fn authorize_ws_rpc_task( - shared: &SharedState, - identity: &AuthenticatedIdentity, - task: &nodedb_physical::physical_task::PhysicalTask, -) -> crate::Result { - let emitter = ArcAuditEmitter(Arc::clone(&shared.audit)); - crate::control::server::shared::authorization::authorize_task_set( - identity, - std::slice::from_ref(task), - &shared.permissions, - &shared.roles, - &emitter, - ) - .map_err(crate::Error::from)? - .into_tasks() - .into_iter() - .next() - .ok_or_else(|| crate::Error::Internal { - detail: "authorization returned an empty capability set".into(), - }) +/// The RPC result for a statement's JSON rows: `null` for none, the row +/// itself for one, an array for more. +fn rows_result(mut rows: Vec) -> serde_json::Value { + match rows.len() { + 0 => serde_json::Value::Null, + 1 => rows.pop().unwrap_or(serde_json::Value::Null), + _ => serde_json::Value::Array(rows), + } } /// Meter one task's dispatch after its rows are pushed onto `results` — diff --git a/nodedb/src/control/server/native/dispatch/cluster_array.rs b/nodedb/src/control/server/native/dispatch/cluster_array.rs index 7a66b1bc8..1495fa7de 100644 --- a/nodedb/src/control/server/native/dispatch/cluster_array.rs +++ b/nodedb/src/control/server/native/dispatch/cluster_array.rs @@ -2,9 +2,9 @@ //! `ClusterArray` plan dispatch for the native protocol. //! -//! `sql_loop.rs` and `single_task.rs` intercept a `PhysicalPlan::ClusterArray` -//! task right after their own in-transaction routing gate and delegate to the -//! shared, protocol-neutral core +//! `single_task.rs` intercepts a `PhysicalPlan::ClusterArray` task right +//! after its own in-transaction routing gate and delegates to the shared, +//! protocol-neutral core //! (`shared::cluster_array_dispatch::execute_cluster_array`), then convert //! the outcome into native wire columns/rows or a count-bearing outcome. Metering //! is not applied on this path, matching pgwire's `ClusterArray` diff --git a/nodedb/src/control/server/native/dispatch/direct_ops.rs b/nodedb/src/control/server/native/dispatch/direct_ops.rs index e7514dd63..5a1d0878b 100644 --- a/nodedb/src/control/server/native/dispatch/direct_ops.rs +++ b/nodedb/src/control/server/native/dispatch/direct_ops.rs @@ -218,6 +218,7 @@ pub(crate) async fn handle_direct_op( &mut tasks, tenant_id, ctx.database_id(), + txn_id, TraceId::ZERO, ) .await @@ -243,6 +244,7 @@ pub(crate) async fn handle_direct_op( &mut tasks, tenant_id, ctx.database_id(), + txn_id, TraceId::ZERO, ) .await diff --git a/nodedb/src/control/server/native/dispatch/edge_recon_gate.rs b/nodedb/src/control/server/native/dispatch/edge_recon_gate.rs index e82f61c0a..a042734b6 100644 --- a/nodedb/src/control/server/native/dispatch/edge_recon_gate.rs +++ b/nodedb/src/control/server/native/dispatch/edge_recon_gate.rs @@ -1,101 +1,49 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Implicit-edge OLLP/Calvin routing gate for the native protocol. +//! The implicit-edge OLLP/Calvin gate for native direct ops. //! -//! A dependent predicate (`BulkDelete`/`BulkUpdate`) on an edge-bearing -//! schemaless collection MUST route through the OLLP/Calvin coordinator so the -//! implicit edge tasks are derived from a pre-exec reconnaissance scan and -//! committed ATOMICALLY with the document write. Without this gate the native -//! path classifies such a delete as `SingleShard` and dispatches directly to -//! the Data Plane — leaving mirrored CSR edges dangling. -//! -//! This module exposes a single entry point: [`try_edge_recon_dispatch`], which -//! implements the same two-guard check as the pgwire `execute.rs` gate and -//! returns the native protocol outcome when the gate fires. +//! The gate itself is shared (`shared::statement_exec::try_edge_recon`). +//! This adapter reads the session's transaction state and renders the +//! applied batch as a native frame. use nodedb_types::protocol::NativeResponse; -use crate::control::planner::calvin::{ - dispatch_authorized_dependent_edge_recon, plan_needs_implicit_edge_recon, -}; use crate::control::server::shared::authorization::AuthorizedTaskSet; use crate::control::server::shared::session::TransactionState; +use crate::control::server::shared::statement_exec::{EdgeRecon, try_edge_recon}; use nodedb_physical::physical_task::PhysicalTask; use super::{DispatchCtx, SqlOutcome, error_to_native}; -/// Attempt to route `tasks` through the implicit-edge OLLP/Calvin path. -/// -/// Returns `Some(outcome)` when the gate fires (the task set contains a -/// `BulkDelete`/`BulkUpdate` on an edge-bearing collection that is not inside -/// an explicit transaction block). The -/// caller MUST return this outcome immediately — the tasks have been consumed. -/// -/// Returns `None` when the gate does not fire; the caller proceeds with the -/// normal classify/dispatch path. -/// -/// A genuine catalog I/O error propagates as `Some(Err-shaped SqlOutcome)` so -/// the caller surfaces it correctly — misrouting on a real I/O fault will -/// silently skip edge cleanup (dangling edges). +/// Route `tasks` through the implicit-edge OLLP/Calvin path when the shared +/// gate fires for them. The caller returns a fired outcome at once: the +/// tasks are consumed. pub(super) async fn try_edge_recon_dispatch( ctx: &DispatchCtx<'_>, seq: u64, tasks: Vec, authorized: AuthorizedTaskSet, ) -> EdgeReconResult { - // Guard 1: not inside an explicit transaction block. The native - // `handle_begin`/`handle_commit`/`handle_rollback` helpers drive the SAME - // `SessionStore` transaction-state machine that pgwire uses, so this guard - // is identical across both protocol paths. Edge-bearing predicate writes - // inside an explicit native transaction block are NOT recon-routed (same - // limitation as pgwire — buffering a multi-step OLLP inside an explicit txn - // will require full two-phase commit across the outer txn boundary). - if ctx.sessions.transaction_state(ctx.peer_addr) == TransactionState::InBlock { - return EdgeReconResult::NotFired(tasks, authorized); - } - - // Guard 2: at least one BulkDelete/BulkUpdate targets an edge-bearing - // collection. A genuine catalog I/O error propagates rather than falling - // through — misrouting on a real fault will skip edge cleanup. - let (_coll, database_id) = - match plan_needs_implicit_edge_recon(ctx.state, &tasks, ctx.tenant_id()) { - Err(e) => return EdgeReconResult::Outcome(resp(error_to_native(seq, &e))), - Ok(None) => return EdgeReconResult::NotFired(tasks, authorized), - Ok(Some(pair)) => pair, - }; - - // Capture the batch's plans before `tasks` is moved into the recon call, so - // the response can shape a RETURNING dependent delete/update's rows and read - // a plain write's affected count from the applied Response the coordinator - // drains. - let plans: Vec<_> = tasks.iter().map(|t| t.plan.clone()).collect(); - - // Both guards passed — run the OLLP/Calvin coordinator. - let outcome = dispatch_authorized_dependent_edge_recon( - ctx.state, - authorized, - ctx.identity, - ctx.tenant_id(), - database_id, - ) - .await; - - EdgeReconResult::Outcome(match outcome { - Ok(recon) => { - // A RETURNING dependent write surfaces its deleted/updated rows; a - // plain write reports the affected count its own mutation returned. - resp(super::conversion::calvin_native_response( + // The native `handle_begin` / `handle_commit` / `handle_rollback` drive + // the same `SessionStore` state machine pgwire uses. + let in_txn_block = ctx.sessions.transaction_state(ctx.peer_addr) == TransactionState::InBlock; + match try_edge_recon(ctx.state, ctx.identity, in_txn_block, tasks, authorized).await { + Ok(EdgeRecon::NotFired(tasks, authorized)) => EdgeReconResult::NotFired(tasks, authorized), + // A RETURNING dependent write surfaces its deleted or updated rows. A + // plain write reports the count its own mutation returned. + Ok(EdgeRecon::Applied(applied)) => { + EdgeReconResult::Outcome(resp(super::conversion::calvin_native_response( seq, - recon.apply_result, - &plans, + applied.apply_result, + &applied.plans, ctx.state, - database_id, + applied.database_id, ctx.tenant_id(), ctx.auth_context(), - )) + ))) } - Err(e) => resp(error_to_native(seq, &e)), - }) + Err(error) => EdgeReconResult::Outcome(resp(error_to_native(seq, &error))), + } } /// Result returned by [`try_edge_recon_dispatch`]. diff --git a/nodedb/src/control/server/native/dispatch/mod.rs b/nodedb/src/control/server/native/dispatch/mod.rs index 64f4bcd72..c691fa0ef 100644 --- a/nodedb/src/control/server/native/dispatch/mod.rs +++ b/nodedb/src/control/server/native/dispatch/mod.rs @@ -23,7 +23,6 @@ mod single_task; mod sorted_read_op; mod sql; mod sql_admin; -mod sql_dispatch_task; mod sql_fold; mod sql_gateway; mod sql_loop; diff --git a/nodedb/src/control/server/native/dispatch/single_task.rs b/nodedb/src/control/server/native/dispatch/single_task.rs index 250705625..86f88cd69 100644 --- a/nodedb/src/control/server/native/dispatch/single_task.rs +++ b/nodedb/src/control/server/native/dispatch/single_task.rs @@ -30,8 +30,8 @@ use super::{ /// the single-shard edge loop share one code path. /// /// Routes through the same protocol-neutral in-transaction staging gate -/// (`route_in_tx_write`) the SQL-planned dispatch loops (`sql_loop.rs`, -/// pgwire's `execute_dml_hooks.rs`) already use. Outside a transaction block +/// (`route_in_tx_write`) the SQL-planned dispatch loops (the shared +/// `statement_exec` loop, pgwire's `execute_dml_hooks.rs`) already use. Outside a transaction block /// the task comes back unchanged (`InTxnRoute::Read`, or `Autocommit` for a /// write), and the gateway gives a write its durable route. Inside a /// transaction block, a stageable write (e.g. `KvBatchPut`) @@ -63,7 +63,7 @@ pub(super) async fn dispatch_single_task( // Covers the `Read` / `Autocommit` dispatch below (reads, and writes that // apply now). `Staged` meters itself inside // `staging_gate::stage_write` — the single choke-point every `Staged` - // route (this file, `sql_loop.rs`, the expander's per-op staging, and + // route (this file, the shared `statement_exec` loop, the expander's per-op staging, and // pgwire's `execute_dml_hooks.rs`) dispatches through, so it is metered // there once rather than duplicated in every caller's closure. `Buffered` // performs no dispatch at all here — it is metered at COMMIT replay diff --git a/nodedb/src/control/server/native/dispatch/sql_dispatch_task.rs b/nodedb/src/control/server/native/dispatch/sql_dispatch_task.rs deleted file mode 100644 index 78a07f960..000000000 --- a/nodedb/src/control/server/native/dispatch/sql_dispatch_task.rs +++ /dev/null @@ -1,142 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! The single-task dispatch primitive the native SQL loop calls. -//! -//! Decides HOW one already-planned task reaches an engine — Control-Plane -//! orchestrators, Exchange resolution, then the gateway — while `sql_loop.rs` -//! decides what to do with the answers. - -use nodedb_types::TraceId; - -use crate::bridge::envelope::Response; -use crate::control::server::exchange::DistributedReadCapture; -use crate::control::server::exchange::resolve::{Resolved, resolve_and_materialize}; -use crate::types::{Lsn, VShardId}; -use nodedb_physical::physical_task::PhysicalTask; - -use super::DispatchCtx; -use super::sql_gateway::dispatch_task_via_gateway; - -/// Dispatch a single `PhysicalTask`, returning the response plus per-shard -/// watermark LSNs a single-node fan gather observed. -/// -/// Multi-arm writes run via Control-Plane orchestrators; everything else -/// routes through `dispatch_task_via_gateway`. -pub(super) async fn dispatch_task( - ctx: &DispatchCtx<'_>, - mut task: PhysicalTask, -) -> crate::Result<(Response, Vec<(VShardId, Lsn)>, Vec)> { - if let crate::bridge::envelope::PhysicalPlan::Document( - nodedb_physical::physical_plan::DocumentOp::InsertSelect { .. }, - ) = &task.plan - { - let authorized = super::sql_gateway::authorize_native_task(ctx, &task)?; - let resp = - crate::control::insert_select::run_authorized_insert_select(ctx.state, authorized) - .await?; - return Ok((resp, Vec::new(), Vec::new())); - } - - // Autocommit `MERGE` orchestrates on the Control Plane (`control::merge_orchestrator`). - if let crate::bridge::envelope::PhysicalPlan::Document( - nodedb_physical::physical_plan::DocumentOp::Merge { - target_collection: _, - source_collection: _, - source_alias: _, - target_join_col: _, - source_join_col: _, - clauses: _, - returning: _, - resolved_inserts: None, - resolved_insert_identities: _, - source_rows: _, - rls_filters: _, - rls_write_check: _, - resolved_sum_targets: _, - declared_primary_key: _, - }, - ) = &task.plan - { - let authorized = super::sql_gateway::authorize_native_task(ctx, &task)?; - let resp = - crate::control::merge_orchestrator::run_authorized_merge(ctx.state, authorized).await?; - return Ok((resp, Vec::new(), Vec::new())); - } - - // Autocommit `UPDATE ... FROM ` scans the source on its own core and - // ships it into the plan, since the source's vShard can live on a different core. - if let crate::bridge::envelope::PhysicalPlan::Document( - nodedb_physical::physical_plan::DocumentOp::UpdateFromJoin { - target_collection: _, - source_collection: _, - source_alias: _, - target_join_col: _, - source_join_col: _, - updates: _, - target_filters: _, - returning: _, - source_rows: None, - rls_filters: _, - rls_write_check: _, - resolved_sum_targets: _, - declared_primary_key: _, - }, - ) = &task.plan - { - let authorized = super::sql_gateway::authorize_native_task(ctx, &task)?; - let resp = crate::control::update_from_join_orchestrator::run_authorized_update_from_join( - ctx.state, authorized, - ) - .await?; - return Ok((resp, Vec::new(), Vec::new())); - } - - // A governed predicate resolves to a concrete row set before proposing - // (`control::write_resolve`). - if let Some(resolver) = crate::control::write_resolve::resolver_for_plan(&task.plan) { - let authorized = super::sql_gateway::authorize_native_task(ctx, &task)?; - let resp = crate::control::write_resolve::run_authorized_write_resolve( - ctx.state, authorized, resolver, - ) - .await?; - return Ok((resp, Vec::new(), Vec::new())); - } - - // Array DDL proposes a replicated catalog entry, as on pgwire. - if crate::control::array_catalog::ddl::is_array_ddl(&task.plan) { - let authorized = super::sql_gateway::authorize_native_task(ctx, &task)?; - let resp = - crate::control::array_catalog::ddl::run_authorized_array_ddl(ctx.state, authorized) - .await?; - return Ok((resp, Vec::new(), Vec::new())); - } - - // Materialize catalog providers and resolve Exchange nodes before dispatch. - // The native protocol has no weaker read consistency: every read is - // strong, so every leg confirms its group where it is served. - let scope = crate::control::server::exchange::ReadScope { - database_id: task.database_id, - tenant_id: task.tenant_id, - trace_id: TraceId::ZERO, - txn_id: task.txn_id, - linearizable: true, - }; - match resolve_and_materialize(ctx.state, ctx.identity, task.plan, scope).await? { - Resolved::Gathered(resp, shard_watermarks, dist_reads) => { - return Ok((resp, shard_watermarks, dist_reads)); - } - Resolved::Plan(resolved_plan) => { - let resolved_plan = *resolved_plan; - task.plan = resolved_plan; - } - // Native path materializes the stream into a Response, preserving gather-then-return. - Resolved::Stream(s) => { - let resp = crate::control::server::exchange::gather::stream_to_response(s).await?; - return Ok((resp, Vec::new(), Vec::new())); - } - } - - // Everything else routes through the gateway. - let resp = dispatch_task_via_gateway(ctx, task).await?; - Ok((resp, Vec::new(), Vec::new())) -} diff --git a/nodedb/src/control/server/native/dispatch/sql_fold.rs b/nodedb/src/control/server/native/dispatch/sql_fold.rs index 413f6faf8..3228fcfa8 100644 --- a/nodedb/src/control/server/native/dispatch/sql_fold.rs +++ b/nodedb/src/control/server/native/dispatch/sql_fold.rs @@ -1,165 +1,67 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Folding each task's answer into one native SQL statement's response: its -//! columns and rows, its warnings, and its one command tag. +//! Rendering one native SQL statement's answer: its columns and rows, its +//! warnings, and its one command tag, or the error frame it stopped with. use nodedb_types::protocol::NativeResponse; -use nodedb_types::value::Value; -use crate::bridge::envelope::PhysicalPlan; -use crate::control::sequence::SessionSequenceAccess; -use crate::control::server::response_shape::compose::{ShapeOutcome, shape_response_materialized}; -use crate::control::server::response_shape::redaction::QueryRedaction; -use crate::control::server::response_shape::request::MaterializedShapeRequest; -use crate::control::server::response_shape::schema::OutputSchema; -use crate::control::server::response_shape::types::{ - DmlOutcome, FoldedTag, PlanKind, StatementTag, describe_plan, staged_dml_outcome, -}; -use crate::control::server::shared::session::staging_gate::{StagedTagKind, StagedWriteOutcome}; -use crate::types::DatabaseId; -use nodedb_physical::physical_task::PhysicalTask; +use crate::control::server::native::sqlstate_code::sqlstate_error; +use crate::control::server::response_shape::types::FoldedTag; +use crate::control::server::shared::statement_exec::{StatementAnswer, StatementError}; use super::{ - DispatchCtx, apply_dml_outcome, dml_fold_error_to_native, shape_error_to_native, - to_native_columns_rows, + apply_dml_outcome, dml_fold_error_to_native, error_code_to_native, error_response_to_native, + error_to_native, error_to_native_with_sqlstate, shape_error_to_native, to_native_columns_rows, }; -/// What shaping a task's answer reads. -pub(super) struct FoldShape<'a> { - pub seq: u64, - pub output_schema: Option<&'a OutputSchema>, - pub database_id: DatabaseId, - /// The concrete session access, never `&dyn SequenceAccess`: the trait - /// object is not `Sync`, and the shape lives across the dispatch awaits - /// of a `Send` connection future. - pub sequences: &'a SessionSequenceAccess<'a>, -} - -/// One statement's response, accumulated across its tasks. -pub(super) struct StatementFold { - pub columns: Option>, - pub rows: Vec>, - pub warnings: Vec, - pub tag: StatementTag, - pub last_lsn: u64, +/// The native frame for a statement's answer or error. +pub(super) fn statement_result_to_native( + seq: u64, + result: Result, +) -> NativeResponse { + match result { + Ok(answer) => answer_to_native(seq, answer), + Err(error) => statement_error_to_native(seq, error), + } } -impl StatementFold { - /// An empty fold for the statement that planned `tasks`. The tag is built - /// over every task, so a derived write beside the user's own adds no - /// verb and no count. - pub(super) fn for_tasks(tasks: &[PhysicalTask]) -> Self { - Self { - columns: None, - rows: Vec::new(), - warnings: Vec::new(), - tag: StatementTag::for_plans(tasks.iter().map(|t| &t.plan)), - last_lsn: 0, +/// The native frame for a statement's answer. The columns are the first +/// non-empty set a task's rows named. +pub(super) fn answer_to_native(seq: u64, answer: StatementAnswer) -> NativeResponse { + let mut r = NativeResponse::ok(seq); + r.watermark_lsn = answer.last_lsn; + r.warnings = answer.warnings; + let mut columns = None; + let mut rows = Vec::new(); + for shaped in &answer.rows { + let (task_columns, task_rows) = to_native_columns_rows(shaped); + if !task_columns.is_empty() && columns.is_none() { + columns = Some(task_columns); } + rows.extend(task_rows); } - - /// Fold the count-bearing outcome of the task that ran `plan` into the - /// statement's tag. - pub(super) fn fold_dml( - &mut self, - seq: u64, - plan: &PhysicalPlan, - outcome: DmlOutcome, - ) -> Result<(), Box> { - self.tag - .fold(self.tag.role_of(plan), outcome) - .map_err(|e| Box::new(dml_fold_error_to_native(seq, &e))) + if !rows.is_empty() { + r.columns = columns; + r.rows = Some(rows); } - - /// Shape `payload`, the answer `plan` gave as `plan_kind`, and fold its - /// rows. Returns the row count, `None` when the payload carries no rows. - pub(super) fn fold_rows( - &mut self, - ctx: &DispatchCtx<'_>, - shape: &FoldShape<'_>, - plan: &PhysicalPlan, - plan_kind: PlanKind, - payload: &[u8], - ) -> Result, Box> { - let redaction = QueryRedaction::for_plan(ctx.tenant_id(), ctx.auth_context(), plan); - match shape_response_materialized(MaterializedShapeRequest { - payload, - plan, - plan_kind, - projection: shape.output_schema, - state: ctx.state, - database_id: shape.database_id, - tenant_id: ctx.tenant_id(), - redaction: Some(redaction.ctx(&ctx.state.redaction)), - sequences: Some(shape.sequences), - }) { - Ok(ShapeOutcome::Rows(mut shaped)) => { - if let Some(notice) = shaped.notice.take() { - self.warnings.push(notice); - } - let (cols, rows) = to_native_columns_rows(&shaped); - if !cols.is_empty() && self.columns.is_none() { - self.columns = Some(cols); - } - let count = rows.len() as u64; - self.rows.extend(rows); - Ok(Some(count)) - } - Ok(ShapeOutcome::Passthrough) => Ok(None), - Err(e) => Err(Box::new(shape_error_to_native(shape.seq, &e))), - } - } - - /// Fold a write its transaction staged. A write that answers `RETURNING` - /// folds its rows in place of a count, as an autocommit one does. A - /// computed-value payload (`RawPayload`) is shaped instead of counted. - pub(super) fn fold_staged( - &mut self, - ctx: &DispatchCtx<'_>, - shape: &FoldShape<'_>, - plan: &PhysicalPlan, - outcome: StagedWriteOutcome, - ) -> Result<(), Box> { - if !outcome.returning_rows.is_empty() { - for rows in &outcome.returning_rows { - self.fold_rows(ctx, shape, plan, PlanKind::ReturningRows, rows)?; - } - return Ok(()); - } - if !matches!(outcome.kind, StagedTagKind::RawPayload) { - return self.fold_dml( - shape.seq, - plan, - staged_dml_outcome(outcome.kind, outcome.affected), - ); - } - if outcome.payload.is_empty() { - self.tag.fold_opaque(); - return Ok(()); - } - // The value rides in the payload, never as a count. - if self - .fold_rows(ctx, shape, plan, describe_plan(plan), &outcome.payload)? - .is_none() - { - self.tag.fold_opaque(); - } - Ok(()) + match answer.tag { + Some(FoldedTag::Dml(outcome)) => apply_dml_outcome(&mut r, outcome), + Some(FoldedTag::Opaque) | None => {} } + r +} - /// The statement's one response. - pub(super) fn finish(self, seq: u64) -> NativeResponse { - let mut r = NativeResponse::ok(seq); - r.watermark_lsn = self.last_lsn; - r.warnings = self.warnings; - if !self.rows.is_empty() { - r.columns = self.columns; - r.rows = Some(self.rows); - } - match self.tag.finish() { - Some(FoldedTag::Dml(outcome)) => apply_dml_outcome(&mut r, outcome), - Some(FoldedTag::Opaque) | None => {} +/// The native error frame for why a statement stopped. +pub(super) fn statement_error_to_native(seq: u64, error: StatementError) -> NativeResponse { + match error { + StatementError::Error(error) => error_to_native(seq, &error), + StatementError::Quota(error) => error_to_native_with_sqlstate(seq, "53400", &error), + StatementError::TenantIsolation => { + sqlstate_error(seq, "42501", "tenant isolation violation") } - r + StatementError::Rejected(code) => error_code_to_native(seq, code.as_ref()), + StatementError::Response(response) => error_response_to_native(seq, &response), + StatementError::DmlFold(error) => dml_fold_error_to_native(seq, &error), + StatementError::Shape(error) => shape_error_to_native(seq, &error), } } diff --git a/nodedb/src/control/server/native/dispatch/sql_gateway.rs b/nodedb/src/control/server/native/dispatch/sql_gateway.rs index 061a31aeb..6ae96a7c1 100644 --- a/nodedb/src/control/server/native/dispatch/sql_gateway.rs +++ b/nodedb/src/control/server/native/dispatch/sql_gateway.rs @@ -1,102 +1,24 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Gateway-based SQL task dispatch for the native protocol. -//! -//! Tasks route through `Gateway::execute_response`, which handles -//! cluster-aware routing, typed `NotLeader` retry, and plan caching. A -//! transaction meta-op runs on its own vShard's core instead. This is native's -//! SQL-TEXT opcode path — distinct from `raw_dispatch.rs`, which serves only -//! native's direct-op opcodes. +//! Task authorization for the native protocol's Control-Plane orchestrated +//! plans. The SQL path's single-task dispatch lives in +//! `shared::statement_exec::dispatch`. -use crate::bridge::envelope::Response; -use std::sync::Arc; - -use crate::control::gateway::core::QueryContext as GatewayQueryContext; -use crate::control::gateway::router::is_task_vshard_scoped; -use crate::control::server::shared::clone_write::CloneCheckedOutcome; -use crate::types::TraceId; use nodedb_physical::physical_task::PhysicalTask; use super::DispatchCtx; -/// Authorize one task with no clone-write check — used only by the -/// Control-Plane orchestrator branches ahead of this file's gateway dispatch, -/// whose plan shapes (`InsertSelect`, `Merge`, `UpdateFromJoin`, a governed -/// predicate resolution, array DDL) are never clone-write shapes. +/// Authorize one task with no clone-write check. Used only by the +/// Control-Plane orchestrator branches, whose plan shapes (`InsertSelect`, +/// `Merge`, `UpdateFromJoin`, a governed predicate resolution, array DDL, a +/// cluster array op, a graph owner op) are never clone-write shapes. pub(super) fn authorize_native_task( ctx: &DispatchCtx<'_>, task: &PhysicalTask, ) -> crate::Result { - let emitter = crate::control::security::audit::ArcAuditEmitter(Arc::clone(&ctx.state.audit)); - crate::control::server::shared::authorization::authorize_task_set( + crate::control::server::shared::statement_exec::authorize_one_task( + ctx.state, ctx.identity, - std::slice::from_ref(task), - &ctx.state.permissions, - &ctx.state.roles, - &emitter, - ) - .map_err(crate::Error::from)? - .into_tasks() - .into_iter() - .next() - .ok_or_else(|| crate::Error::Internal { - detail: "authorization returned an empty capability set".into(), - }) -} - -/// Dispatch a single `PhysicalTask` through the gateway. -/// -/// Returns the Data-Plane `Response` shape, with a `NotFound` verdict as an -/// error status. -pub(super) async fn dispatch_task_via_gateway( - ctx: &DispatchCtx<'_>, - task: PhysicalTask, -) -> crate::Result { - let tenant_id = task.tenant_id; - // Clone CoW write-path interception, then authorization, run once per - // task before dispatch — same protocol-neutral gate raw_dispatch runs. - let emitter = crate::control::security::audit::ArcAuditEmitter(Arc::clone(&ctx.state.audit)); - let checked = match crate::control::server::shared::clone_write::intercept_and_authorize( - crate::control::server::shared::clone_write::InterceptAndAuthorizeParams { - state: ctx.state, - task, - identity: ctx.identity, - tenant_id, - permissions: &ctx.state.permissions, - roles: &ctx.state.roles, - emitter: &emitter, - }, + task, ) - .await? - { - CloneCheckedOutcome::Handled(resp) => return Ok(resp), - CloneCheckedOutcome::Proceed(checked) => checked, - }; - let database_id = checked.database_id(); - let txn_id = checked.txn_id(); - - let gateway = ctx.state.installed_gateway()?; - // A staged write and the other transaction meta-ops run on the core of - // the task's own vShard. The gateway will route them to vShard 0. A - // write takes the durable route, a read the read route. - if is_task_vshard_scoped(checked.plan()) { - return crate::control::server::dispatch_utils::dispatch_authorized_task_by_class( - ctx.state, - checked, - TraceId::generate(), - ) - .await; - } - let gw_ctx = GatewayQueryContext { - tenant_id, - trace_id: TraceId::generate(), - database_id, - // Propagate the in-block transaction id so gateway local dispatch - // resolves the per-txn staging overlay. - txn_id, - linearizable: true, - }; - // The typed error passes through unchanged. The native frame renders its - // SQLSTATE and numeric code from it. - gateway.execute_response(&gw_ctx, checked).await } diff --git a/nodedb/src/control/server/native/dispatch/sql_loop.rs b/nodedb/src/control/server/native/dispatch/sql_loop.rs index fbd0c2ffc..cb5836f77 100644 --- a/nodedb/src/control/server/native/dispatch/sql_loop.rs +++ b/nodedb/src/control/server/native/dispatch/sql_loop.rs @@ -1,51 +1,26 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Per-task dispatch loop for the DataFusion-planned SQL path. +//! The native SQL path's per-task loop: the shared statement loop +//! (`shared::statement_exec`), rendered as one native frame. //! //! A statement in a transaction (the client's block, or the implicit //! transaction a statement whose writes fire a BEFORE, INSTEAD OF or SYNC //! AFTER body runs in) routes each task through the shared `txn_route`, the -//! route pgwire takes too: the write's trigger bodies join the transaction, -//! a shadowed clone takes its copy-on-write steps, and the write stages. The -//! single-task dispatch primitive lives in `sql_dispatch_task.rs`, and the -//! folding of each task's answer in `sql_fold.rs`. +//! route pgwire takes too. The frame renders in `sql_fold.rs`. -use std::ops::ControlFlow; use std::sync::Arc; -use nodedb_types::protocol::NativeResponse; - -use crate::bridge::envelope::Status; use crate::control::sequence::SessionSequenceAccess; use crate::control::server::response_shape::schema::OutputSchema; -use crate::control::server::response_shape::types::{ - PlanKind, TaskTagRole, describe_plan, payload_to_dml_outcome, replaced_write_outcome, -}; -use crate::control::server::shared::metering::{PlanMeteringInfo, meter_dispatch}; -use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; use crate::control::server::shared::session::DmlTxnCtx; use crate::control::server::shared::session::read_set::ReadSetEntry; -use crate::control::server::shared::session::staging_gate::StagingGateError; -use crate::control::server::shared::txn_route::{ - StatementEvents, TxnTaskContext, TxnTaskOutcome, route_txn_task, -}; +use crate::control::server::shared::statement_exec::{self, StatementExec}; use crate::types::DatabaseId; use nodedb_physical::physical_task::PhysicalTask; -use super::sql_dispatch_task::dispatch_task; -use super::sql_fold::{FoldShape, StatementFold}; +use super::DispatchCtx; +use super::sql_fold::statement_result_to_native; use super::streaming::SqlOutcome; -use super::{ - DispatchCtx, error_code_to_native, error_response_to_native, error_to_native, - error_to_native_with_sqlstate, -}; -use crate::control::server::native::sqlstate_code::sqlstate_error; - -/// Wrap a materialized response as a non-streaming [`SqlOutcome`]. -#[inline] -fn resp(r: NativeResponse) -> SqlOutcome { - SqlOutcome::Response(Box::new(r)) -} /// A planned statement the loop dispatches. pub(super) struct PlannedStatement<'a> { @@ -58,18 +33,23 @@ pub(super) struct PlannedStatement<'a> { pub sum_target_reads: Vec, } -/// Run the per-task dispatch loop for a planned, non-streamed task set, -/// materializing all rows/columns/affected-count into a single -/// [`SqlOutcome::Response`]. `txn` is the statement's transaction, `None` +/// The session's sequence access for a statement in `database_id`: the +/// registry plus this connection's `currval` map. +pub(super) fn session_sequences<'a>( + ctx: &DispatchCtx<'a>, + database_id: DatabaseId, +) -> SessionSequenceAccess<'a> { + SessionSequenceAccess::for_session( + ctx.state, + ctx.sessions.sequence_values(ctx.peer_addr), + database_id, + ctx.tenant_id(), + ) +} + +/// Run the per-task loop for a planned, non-streamed task set, rendered as +/// one [`SqlOutcome::Response`]. `txn` is the statement's transaction, `None` /// for an autocommit statement. -/// -/// The statement's `rows_affected` and `command` come from one -/// [`crate::control::server::response_shape::types::StatementTag`] folded -/// over every task, the same fold pgwire renders as its command tag: a -/// count-bearing task contributes its reported count and verb, an opaque task -/// (buffered write, index or graph maintenance, computed-value payload) -/// contributes nothing. Neither field is ever synthesised from the task -/// count. pub(super) async fn run_dispatch_loop( ctx: &DispatchCtx<'_>, seq: u64, @@ -83,245 +63,26 @@ pub(super) async fn run_dispatch_loop( plan_lease_scope, sum_target_reads, } = statement; - if let Some(txn) = txn - && !sum_target_reads.is_empty() - { - txn.sessions - .record_read_entries(txn.session_id, sum_target_reads); - } - // A derived write beside the user's own (an implicit edge, a cross-shard - // balance move) never answers the statement, exactly as Calvin's deposit - // rule has it: the fold is built over every task to decide it. - let mut fold = StatementFold::for_tasks(&tasks); - // Checked once rather than per task — metering is disabled by default, so - // this keeps the per-task extraction below (which clones the collection - // name) a true no-op on the hot path for every deployment that hasn't - // turned it on. - let metering_enabled = ctx.state.metering_config.enabled; - // Session-scoped sequence access for the statement's Control-Plane - // computed columns: the registry plus this connection's `currval` map. - let session_sequences = ctx.sessions.sequence_values(ctx.peer_addr); - let sequences = SessionSequenceAccess::for_session( - ctx.state, - session_sequences, - database_id, - ctx.tenant_id(), - ); - let shape = FoldShape { - seq, + let sequences = session_sequences(ctx, database_id); + let exec = StatementExec { + state: ctx.state, + identity: ctx.identity, + scope: &ctx.scope, output_schema, database_id, sequences: &sequences, }; - let route_ctx = txn.map(|txn| TxnTaskContext { - state: ctx.state, - identity: ctx.identity, - auth: ctx.auth_context(), + let result = statement_exec::run_statement_loop( + &exec, + statement_exec::PlannedStatement { + tasks, + plan_lease_scope, + sum_target_reads, + }, txn, - lease_scope: &plan_lease_scope, - fire_triggers: true, - }); - let mut statement_events = StatementEvents::default(); - - for task in tasks { - if task.tenant_id != ctx.tenant_id() { - return resp(sqlstate_error(seq, "42501", "tenant isolation violation")); - } - - // Cloned before routing consumes `task`: a staged write's rows and - // computed-value payload shape against it. - let plan = task.plan.clone(); - // Metering needs the collection/engine shape after this task's - // dispatch succeeds. It covers the direct dispatch below (reads, and - // writes that apply now). A staged write meters itself inside the - // staging gate, and a buffered write meters at COMMIT. - let plan_metering_info = metering_enabled.then(|| PlanMeteringInfo::extract(&plan)); - - // A spent hard quota refuses the task before it runs; the charging - // call at the end of this loop is on the success path and so can - // never refuse anything itself. - if let Some(info) = &plan_metering_info - && let Err(e) = admit_quota_for_dispatch(ctx.state, &ctx.scope, info) - { - return resp(error_to_native_with_sqlstate(seq, "53400", &e)); - } - - // The task to dispatch below: the input task, or the one routing - // hands back. Every other routing outcome answers the task here. - let task = if let Some(route) = route_ctx.as_ref() { - let routed = route_txn_task( - route, - task, - &mut statement_events, - |stage_task| async move { - dispatch_task(ctx, stage_task) - .await - .map(|(resp, _, _)| resp) - }, - ) - .await; - let answered = match routed { - Ok(TxnTaskOutcome::Dispatch(routed_task)) => ControlFlow::Continue(*routed_task), - Ok(TxnTaskOutcome::Buffered) => { - // Applied at COMMIT: no count and no verb yet. - fold.tag.fold_opaque(); - ControlFlow::Break(Ok(())) - } - Ok(TxnTaskOutcome::Staged(outcome)) => { - ControlFlow::Break(fold.fold_staged(ctx, &shape, &plan, outcome)) - } - Ok(TxnTaskOutcome::CloneHandled(clone_resp)) => ControlFlow::Break( - fold_answer(ctx, &shape, &mut fold, &plan, clone_resp.payload.as_ref()) - .map(drop), - ), - Ok(TxnTaskOutcome::InsteadOf) => { - ControlFlow::Break(match replaced_write_outcome(describe_plan(&plan)) { - Some(outcome) => fold.fold_dml(seq, &plan, outcome), - None => { - fold.tag.fold_opaque(); - Ok(()) - } - }) - } - Err(StagingGateError::Dispatch(e)) => return resp(error_to_native(seq, &e)), - Err(StagingGateError::Rejected { code }) => { - return resp(error_code_to_native(seq, code.as_ref())); - } - }; - match answered { - ControlFlow::Continue(routed_task) => routed_task, - ControlFlow::Break(Ok(())) => continue, - ControlFlow::Break(Err(e)) => return SqlOutcome::Response(e), - } - } else { - task - }; - - // `ClusterArray` plans are handled entirely on the Control Plane by - // the `ArrayCoordinator` — they must never reach the SPSC bridge or - // the trigger/DML machinery. A write in a transaction reshapes into - // per-shard `ArrayOp` tasks inside the staging gate above and never - // reaches here as `ClusterArray`; only reads (`Slice`/`Agg`) and - // autocommit `Put`/`Delete` arrive as this plan. Intercepted here, - // before `dispatch_task` will otherwise route it through the gateway - // toward the Data Plane. Metering is not applied here, matching - // pgwire's `ClusterArray` short-circuit, which also does not meter - // this path. - if matches!( - task.plan, - crate::bridge::envelope::PhysicalPlan::ClusterArray(_) - ) { - let authorized = match super::sql_gateway::authorize_native_task(ctx, &task) { - Ok(a) => a, - Err(e) => return resp(error_to_native(seq, &e)), - }; - match super::cluster_array::dispatch_cluster_array_task(ctx, authorized, output_schema) - .await - { - Ok(super::cluster_array::ClusterArrayOutcome::Rows { - columns, - rows, - notice, - }) => { - if let Some(n) = notice { - fold.warnings.push(n); - } - if !columns.is_empty() && fold.columns.is_none() { - fold.columns = Some(columns); - } - fold.rows.extend(rows); - } - Ok(super::cluster_array::ClusterArrayOutcome::Affected(outcome)) => { - if let Err(e) = fold.fold_dml(seq, &plan, outcome) { - return SqlOutcome::Response(e); - } - } - Err(e) => return resp(error_to_native(seq, &e)), - } - continue; - } - - let task_vshard = task.vshard_id; - let task_database_id = task.database_id; - let (task_resp, shard_watermarks, dist_reads) = match dispatch_task(ctx, task).await { - Ok(r) => r, - Err(e) => return resp(error_to_native(seq, &e)), - }; - - // Track reads for snapshot-isolation / cross-shard conflict detection - // at the protocol-neutral layer, into the statement's transaction. - // Recorded BEFORE the error short-circuit so an absent-key point read - // (a `NotFound` from the Data Plane) is captured too; a "not found" is - // a validatable phantom observation. A multi-core fan read records one - // entry per participating shard from the gather's per-shard - // watermarks; a single read falls back to its one watermark. - let records_read = task_resp.status == Status::Ok - || task_resp.error_code.as_deref() - == Some(&crate::bridge::envelope::ErrorCode::NotFound); - if records_read && let Some(txn) = txn { - let watermarks = if shard_watermarks.is_empty() { - vec![(task_vshard, task_resp.watermark_lsn)] - } else { - shard_watermarks - }; - crate::control::server::shared::session::record_reads_for_response( - ctx.state, - txn.sessions, - txn.session_id, - ctx.tenant_id(), - crate::control::server::shared::session::ResponseReads { - plan: &plan, - watermarks: &watermarks, - read_version_lsn: task_resp.read_version_lsn, - found: task_resp.status == Status::Ok, - distributed_reads: &dist_reads, - read_lsn_vshard: task_vshard, - }, - ) - .await; - } - - if task_resp.status == Status::Error { - return resp(error_response_to_native(seq, &task_resp)); - } - - // --- TRUNCATE RESTART IDENTITY --- - // Autocommit only: a buffered truncate restarts its sequences at - // COMMIT. Same rule as the pgwire dispatch loop. - if let Some((collection, true)) = plan.truncate_target() { - ctx.state - .sequence_registry - .restart_sequences_for_collection( - task_database_id.as_u64(), - ctx.tenant_id().as_u64(), - collection.as_str(), - ); - } - - fold.last_lsn = task_resp.watermark_lsn.as_u64(); - - let task_rows = match fold_answer(ctx, &shape, &mut fold, &plan, task_resp.payload.as_ref()) - { - Ok(rows) => rows, - Err(e) => return SqlOutcome::Response(e), - }; - - // Metered here, once per successfully dispatched task (see the scope - // note on `plan_metering_info` above for the tasks this does not - // cover) — `task_resp.status == Status::Error` returned above, so - // every path reaching here is the success path. - if let Some(info) = &plan_metering_info { - meter_dispatch(ctx.state, &ctx.scope, info, task_rows); - } - } - - if let Some(route) = route_ctx.as_ref() - && let Err(e) = statement_events.fire(route).await - { - return resp(error_to_native(seq, &e)); - } - - resp(fold.finish(seq)) + ) + .await; + SqlOutcome::Response(Box::new(statement_result_to_native(seq, result))) } /// Run a statement in an implicit transaction: it commits when every task @@ -331,73 +92,35 @@ pub(super) async fn run_implicit_statement( seq: u64, statement: PlannedStatement<'_>, ) -> SqlOutcome { + let PlannedStatement { + tasks, + output_schema, + database_id, + plan_lease_scope, + sum_target_reads, + } = statement; + let sequences = session_sequences(ctx, database_id); + let exec = StatementExec { + state: ctx.state, + identity: ctx.identity, + scope: &ctx.scope, + output_schema, + database_id, + sequences: &sequences, + }; let client = DmlTxnCtx { sessions: ctx.sessions, session_id: ctx.peer_addr.into(), }; - let outcome = crate::control::trigger::statement_txn::with_statement_txn_lifted( - ctx.state, - ctx.identity, - &client, - true, - |error| resp(error_to_native(seq, &error)), - async |txn: &DmlTxnCtx<'_>| { - let outcome = run_dispatch_loop(ctx, seq, statement, Some(txn)).await; - match &outcome { - SqlOutcome::Response(r) - if r.status == nodedb_types::protocol::ResponseStatus::Error => - { - Err(outcome) - } - _ => Ok(outcome), - } + let result = statement_exec::run_implicit_statement( + &exec, + statement_exec::PlannedStatement { + tasks, + plan_lease_scope, + sum_target_reads, }, + &client, ) .await; - match outcome { - Ok(outcome) | Err(outcome) => outcome, - } -} - -/// Fold a dispatched task's answer `payload`: its rows, or its count. Returns -/// the task's own row count for metering, `None` when it carries none. -fn fold_answer( - ctx: &DispatchCtx<'_>, - shape: &FoldShape<'_>, - fold: &mut StatementFold, - plan: &crate::bridge::envelope::PhysicalPlan, - payload: &[u8], -) -> Result, Box> { - let plan_kind = describe_plan(plan); - let count_bearing = matches!(plan_kind, PlanKind::DmlResult(_) | PlanKind::DmlResultByOp); - if payload.is_empty() && !count_bearing { - // Not a count-bearing plan (graph / vector / index write): no count - // and no verb to report. A count-bearing plan with no payload falls - // through so the count reader refuses it. - fold.tag.fold_opaque(); - return Ok(None); - } - if let Some(rows) = fold.fold_rows(ctx, shape, plan, plan_kind, payload)? { - return Ok(Some(rows)); - } - // A count-bearing write reports the rows it touched and its verb; an - // opaque execution reports neither. Counting one per dispatched task - // instead will report a row for a delete that removed nothing and for an - // `ON CONFLICT DO NOTHING` insert that skipped. - if fold.tag.role_of(plan) == TaskTagRole::Opaque { - fold.tag.fold_opaque(); - return Ok(None); - } - match payload_to_dml_outcome(payload, plan_kind) { - Ok(Some(outcome)) => { - let affected = outcome.affected; - fold.fold_dml(shape.seq, plan, outcome)?; - Ok(Some(affected)) - } - Ok(None) => { - fold.tag.fold_opaque(); - Ok(None) - } - Err(e) => Err(Box::new(error_to_native(shape.seq, &e))), - } + SqlOutcome::Response(Box::new(statement_result_to_native(seq, result))) } diff --git a/nodedb/src/control/server/native/dispatch/sql_planned.rs b/nodedb/src/control/server/native/dispatch/sql_planned.rs index 7ba94e01b..93b0d7522 100644 --- a/nodedb/src/control/server/native/dispatch/sql_planned.rs +++ b/nodedb/src/control/server/native/dispatch/sql_planned.rs @@ -8,26 +8,22 @@ use nodedb_types::protocol::NativeResponse; use std::sync::Arc; -use crate::control::planner::calvin::{ - CrossShardTxnMode, DispatchClass, TxnDispatchPosition, classify_dispatch, - dispatch_authorized_tasks_to_calvin, -}; -use crate::control::security::audit::ArcAuditEmitter; use crate::control::server::native::sqlstate_code::sqlstate_error; use crate::control::server::shared::check_constraint::{ enforce_statement_checks, enforce_statement_enum_labels, }; use crate::control::server::shared::plan_admission::{ - PlanAdmissionRequest, plan_authorize_and_admit, + PlanAdmissionRequest, plan_authorize_and_admit_in_txn, }; use crate::control::server::shared::session::{DmlTxnCtx, TransactionState}; -use crate::control::server::shared::txn_route::{ - statement_needs_implicit_txn, unbufferable_joined_statement, +use crate::control::server::shared::statement_exec::{ + AtomicRoute, AtomicStatement, StatementExec, route_atomic_statement, }; -use crate::control::server::shared::write_admission::{all_writes_bufferable, plan_is_write}; +use crate::control::server::shared::write_admission::plan_is_write; use super::sql::resp; -use super::sql_loop::{PlannedStatement, run_dispatch_loop, run_implicit_statement}; +use super::sql_fold::{answer_to_native, statement_error_to_native}; +use super::sql_loop::{PlannedStatement, run_dispatch_loop, session_sequences}; use super::streaming::{SqlOutcome, try_open_sql_stream}; use super::{DispatchCtx, ddl_result_to_native, error_to_native}; @@ -100,158 +96,86 @@ pub(super) async fn execute_planned( // still follows authorization inside the unit, so denied requests consume // no descriptor lease. The scope stays alive through all dispatch and // response shaping below. - let admission = match plan_authorize_and_admit(PlanAdmissionRequest { - state: ctx.state, - query_ctx: ctx.query_ctx, - scope: &scope, - sql: &clean_sql, - trace_id: TraceId::ZERO, - }) + // The derived writes read their source rows through the open + // transaction's staging overlay, so they see this transaction's earlier + // statements. + let admission = match plan_authorize_and_admit_in_txn( + PlanAdmissionRequest { + state: ctx.state, + query_ctx: ctx.query_ctx, + scope: &scope, + sql: &clean_sql, + trace_id: TraceId::ZERO, + }, + ctx.sessions.tx_id(ctx.peer_addr), + ) .await { Ok(admission) => admission, Err(error) => return resp(error_to_native(seq, &error)), }; - let mut tasks = admission.tasks; let output_schema = admission.output_schema; - // Re-derived here rather than carried from `admission`: Calvin dispatch - // and implicit-edge reconciliation are trusted internal mechanisms that - // never reach the clone-checked dispatch boundary (they don't call - // `dispatch_authorized_to_data_plane` / `Gateway::execute`), so a plain - // batch authorize is correct for them. `run_dispatch_loop` below clone-checks - // and authorizes each task itself, immediately before its own dispatch. - let mut authorized_tasks = - match crate::control::server::shared::authorization::authorize_task_set( - ctx.identity, - &tasks, - &ctx.state.permissions, - &ctx.state.roles, - &ArcAuditEmitter(Arc::clone(&ctx.state.audit)), - ) { - Ok(authorized) => authorized, - Err(error) => return resp(error_to_native(seq, &crate::Error::from(error))), - }; - let mut lease_scope = Some(admission.lease_scope); - // Covers the images every cross-shard materialized-sum balance in `tasks` - // was settled from, so Calvin's OCC check aborts rather than committing a - // total folded from an image that has since moved. - let sum_target_reads = admission.sum_target_reads; - - if tasks.is_empty() { + if admission.tasks.is_empty() { return resp(NativeResponse::status_row(seq, "OK")); } - // A statement outside a transaction block whose writes fire a BEFORE, - // INSTEAD OF or SYNC AFTER body, a MERGE into an edge-bearing collection, - // or a join-expanding write into the source of a cross-shard - // materialized sum, runs in an implicit transaction ahead of every autocommit - // route: its writes stage on their vShards' leaders and commit together - // at its end. + // The shared atomic routes, the ones every protocol takes: the implicit + // transaction, the implicit-edge OLLP/Calvin gate, and a Calvin commit + // for a write that spans vShards. The native session's + // `cross_shard_txn` parameter picks the cross-shard mode. An unset value + // is the documented default, `CrossShardTxnMode::Strict`. let in_txn_block = ctx.sessions.transaction_state(ctx.peer_addr) == TransactionState::InBlock; - if !in_txn_block && statement_needs_implicit_txn(ctx.state, &tasks) { - if !all_writes_bufferable(&tasks) { - return resp(error_to_native(seq, &unbufferable_joined_statement())); - } - let Some(lease_scope) = lease_scope.take() else { - return resp(sqlstate_error( + let session = DmlTxnCtx { + sessions: ctx.sessions, + session_id: ctx.peer_addr.into(), + }; + let sequences = session_sequences(ctx, database_id); + let exec = StatementExec { + state: ctx.state, + identity: ctx.identity, + scope: &ctx.scope, + output_schema: Some(&output_schema), + database_id, + sequences: &sequences, + }; + let routed = route_atomic_statement( + &exec, + AtomicStatement { + tasks: admission.tasks, + lease_scope: admission.lease_scope, + // Covers the images every cross-shard materialized-sum balance in + // the tasks was settled from, so Calvin's OCC check aborts rather + // than committing a total folded from an image that has since + // moved. + sum_target_reads: admission.sum_target_reads, + }, + &session, + ) + .await; + let AtomicStatement { + tasks, + lease_scope, + sum_target_reads, + } = match routed { + Ok(AtomicRoute::Implicit(answer)) => return resp(answer_to_native(seq, answer)), + // A RETURNING write surfaces its rows from the applied response. A + // plain write reports the count its own mutation returned. + Ok(AtomicRoute::Calvin(applied)) => { + return resp(super::conversion::calvin_native_response( seq, - nodedb_types::error::sqlstate::INTERNAL_ERROR, - "internal error: query lease scope missing before SQL dispatch", + applied.apply_result, + &applied.plans, + ctx.state, + applied.database_id, + ctx.tenant_id(), + ctx.auth_context(), )); - }; - return run_implicit_statement( - ctx, - seq, - PlannedStatement { - tasks, - output_schema: Some(&output_schema), - database_id, - plan_lease_scope: Arc::new(lease_scope), - sum_target_reads, - }, - ) - .await; - } - - // Implicit-edge DELETE/UPDATE routing gate (native-protocol parity with - // pgwire). See `edge_recon_gate` for the full invariant and guard - // documentation. Returns early when the gate fires, consuming `tasks`. - { - use super::edge_recon_gate::{EdgeReconResult, try_edge_recon_dispatch}; - match try_edge_recon_dispatch(ctx, seq, tasks, authorized_tasks).await { - EdgeReconResult::Outcome(outcome) => return outcome, - EdgeReconResult::NotFired(returned_tasks, returned_authorized) => { - tasks = returned_tasks; - authorized_tasks = returned_authorized; - } } - } - - // Cross-shard write parity with pgwire: classify the planned task set and, - // for a strict multi-shard write, route the whole batch through the Calvin - // sequencer so it commits atomically. Single-shard (and best-effort) keep - // the existing per-task gateway/SPSC dispatch loop below unchanged. - // Autocommit single-statement dispatch: no session read-set to widen with. - let sum_read_vshards = match crate::control::planner::calvin::read_vshards_of(&sum_target_reads) - { - Ok(vshards) => vshards, - Err(error) => return resp(error_to_native(seq, &error)), + Ok(AtomicRoute::PerTask(statement)) => statement, + Err(error) => return resp(statement_error_to_native(seq, error)), }; - match classify_dispatch(&tasks, &sum_read_vshards) { - DispatchClass::SingleShard { .. } => {} - DispatchClass::MultiShard { .. } => { - // Dispatching to Calvin here applies the statement durably at - // statement time, escaping the transaction buffer. Inside a block, - // fall through to the per-task staging gate when the gate can - // buffer every write — COMMIT flushes the whole buffer through - // Calvin. Anything else is refused, matching pgwire. - if in_txn_block && !all_writes_bufferable(&tasks) { - return resp(error_to_native( - seq, - &crate::Error::CrossShardInExplicitTransaction, - )); - } - - // Native has no per-session `cross_shard_txn` parameter wired, so it - // reads the same `SessionStore` accessor pgwire uses; an unset value - // defaults to `CrossShardTxnMode::Strict` (the documented default), - // so native multi-shard writes route through Calvin by default. - let cross_shard_mode = ctx.sessions.cross_shard_txn_mode(ctx.peer_addr); - if !in_txn_block && cross_shard_mode == CrossShardTxnMode::Strict { - return match dispatch_authorized_tasks_to_calvin( - ctx.state, - authorized_tasks, - ctx.tenant_id(), - cross_shard_mode, - TxnDispatchPosition::Autocommit, - &sum_target_reads, - None, - ) - .await - { - // Calvin committed. A RETURNING write surfaces its rows from - // the applied Response; a plain write reports the affected - // count its own mutation returned. - Ok(apply_resp) => { - let plans: Vec<_> = tasks.iter().map(|t| t.plan.clone()).collect(); - resp(super::conversion::calvin_native_response( - seq, - apply_resp, - &plans, - ctx.state, - database_id, - ctx.tenant_id(), - ctx.auth_context(), - )) - } - Err(e) => resp(error_to_native(seq, &e)), - }; - } - // An in-block statement and `BestEffortNonAtomic` both fall - // through to the per-task loop below. - } - } + let mut lease_scope = Some(lease_scope); // A native lazy stream outlives this handler, so transfer its descriptor // leases to the session-owned `SqlStream` before returning it. The session @@ -295,10 +219,6 @@ pub(super) async fn execute_planned( } let read_only = tasks.iter().all(|task| !plan_is_write(&task.plan)); let guard_scope = Arc::clone(&lease_scope); - let client_txn = DmlTxnCtx { - sessions: ctx.sessions, - session_id: ctx.peer_addr.into(), - }; let run = run_dispatch_loop( ctx, seq, @@ -309,7 +229,7 @@ pub(super) async fn execute_planned( plan_lease_scope: lease_scope, sum_target_reads, }, - in_txn_block.then_some(&client_txn), + in_txn_block.then_some(&session), ); if read_only { match guard_scope.guard(run).await { diff --git a/nodedb/src/control/server/pgwire/handler/routing/execute.rs b/nodedb/src/control/server/pgwire/handler/routing/execute.rs index 8ed955abb..3c7c9a272 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/execute.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/execute.rs @@ -80,12 +80,16 @@ impl NodeDbPgHandler { // this pass settled was folded from; they travel on the // dispatch read-set so Calvin's OCC check aborts rather than // committing a total folded from an image that has moved. + // The source rows are read through the open transaction's + // staging overlay, so they include its earlier statements. + let read_txn = self.sessions.tx_id(session_id); let sum_target_reads = crate::control::planner::materialized_sum::resolve_materialized_sum_targets( &self.state, &mut tasks, tenant_id, edge_database_id, + read_txn, crate::types::TraceId::ZERO, ) .await @@ -111,6 +115,7 @@ impl NodeDbPgHandler { &mut tasks, tenant_id, edge_database_id, + read_txn, crate::types::TraceId::ZERO, ) .await diff --git a/nodedb/src/control/server/shared/cluster_array_dispatch.rs b/nodedb/src/control/server/shared/cluster_array_dispatch.rs index 6e065ed54..a816b659d 100644 --- a/nodedb/src/control/server/shared/cluster_array_dispatch.rs +++ b/nodedb/src/control/server/shared/cluster_array_dispatch.rs @@ -7,9 +7,10 @@ //! trigger/DML machinery. Each transport's dispatch loop intercepts a //! `PhysicalPlan::ClusterArray` task after authorization: //! -//! - pgwire and native call [`execute_cluster_array`] and render the -//! [`ClusterArrayShaped`] outcome in their own wire format. The adapters -//! live in `pgwire::handler::routing::cluster_array` and +//! - pgwire, native and the shared statement loop +//! (`shared::statement_exec`) call [`execute_cluster_array`] and render the +//! [`ClusterArrayShaped`] outcome in their own shape. The protocol +//! adapters live in `pgwire::handler::routing::cluster_array` and //! `native::dispatch::cluster_array`. //! - HTTP and WebSocket RPC call [`run_cluster_array`] and shape the raw //! payload the way they shape a gateway payload. diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch.rs index 73730c139..25d020762 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch.rs @@ -128,13 +128,16 @@ pub(in crate::control::server::shared::ddl::neutral::collection) async fn plan_a // The entries cover the row images every cross-shard balance this pass // settled was folded from. They travel on the dispatch read-set so the // Calvin OCC check aborts, before any row moves, if those images have been - // written since. + // written since. The source rows are read through the open transaction's + // staging overlay, so they include this transaction's earlier statements. + let read_txn = txn_ctx.sessions.tx_id(txn_ctx.session_id); let sum_target_reads = crate::control::planner::materialized_sum::resolve_materialized_sum_targets( state, &mut tasks, tenant_id, database_id, + read_txn, TraceId::ZERO, ) .await @@ -159,6 +162,7 @@ pub(in crate::control::server::shared::ddl::neutral::collection) async fn plan_a &mut tasks, tenant_id, database_id, + read_txn, TraceId::ZERO, ) .await diff --git a/nodedb/src/control/server/shared/mod.rs b/nodedb/src/control/server/shared/mod.rs index 6d1f001ad..f4c1a7df8 100644 --- a/nodedb/src/control/server/shared/mod.rs +++ b/nodedb/src/control/server/shared/mod.rs @@ -17,6 +17,7 @@ pub mod retry; pub mod returning; pub mod session; pub mod sql; +pub mod statement_exec; pub mod txn_control; pub mod txn_route; pub mod write_admission; diff --git a/nodedb/src/control/server/shared/plan_admission.rs b/nodedb/src/control/server/shared/plan_admission.rs index 46577c02f..d31666655 100644 --- a/nodedb/src/control/server/shared/plan_admission.rs +++ b/nodedb/src/control/server/shared/plan_admission.rs @@ -74,12 +74,30 @@ pub struct PlanAdmissionRequest<'a> { /// Plan `sql`, authorize it, expand implicit graph edges, authorize the expanded /// set, and acquire the descriptor leases — retrying the whole unit while a /// descriptor drain is in flight. +/// +/// For a transport with no transaction block: every derived read sees +/// committed base only. A session transport uses +/// [`plan_authorize_and_admit_in_txn`]. pub async fn plan_authorize_and_admit( request: PlanAdmissionRequest<'_>, +) -> crate::Result { + plan_authorize_and_admit_in_txn(request, None).await +} + +/// [`plan_authorize_and_admit`] for a statement that runs in the open +/// transaction `read_txn`, `None` outside a transaction block. +/// +/// The derived writes read their source rows through that transaction's +/// staging overlay. A materialized-sum balance or a period lock then derives +/// from the rows as this transaction sees them, including the rows its earlier +/// statements staged. +pub async fn plan_authorize_and_admit_in_txn( + request: PlanAdmissionRequest<'_>, + read_txn: Option, ) -> crate::Result { let request = &request; retry_on_schema_change(&request.state.lease_drain, move || { - plan_authorize_and_admit_once(request) + plan_authorize_and_admit_once(request, read_txn) }) .await } @@ -88,6 +106,7 @@ pub async fn plan_authorize_and_admit( /// re-invocation with no partial state carried between attempts. async fn plan_authorize_and_admit_once( request: &PlanAdmissionRequest<'_>, + read_txn: Option, ) -> crate::Result { let state = request.state; let query_ctx = request.query_ctx; @@ -137,8 +156,15 @@ async fn plan_authorize_and_admit_once( let _preauthorized_tasks = authorize_task_set(identity, &tasks, &state.permissions, &state.roles, &emitter)?; - let sum_target_reads = - append_derived_tasks(state, &mut tasks, tenant_id, database_id, trace_id).await?; + let sum_target_reads = append_derived_tasks( + state, + &mut tasks, + tenant_id, + database_id, + read_txn, + trace_id, + ) + .await?; // Deliberate gate: proves the final task set is authorizable before a // descriptor lease is acquired. The caller re-derives the capability per @@ -166,11 +192,16 @@ async fn plan_authorize_and_admit_once( /// /// Returns the read-set entries covering the images every cross-shard /// balance was settled from. The caller unions them into its read set. +/// +/// `read_txn` is the open transaction the statement runs in, `None` outside a +/// transaction block. Each derived write reads its source rows through that +/// transaction's staging overlay. pub async fn append_derived_tasks( state: &SharedState, tasks: &mut Vec, tenant_id: crate::types::TenantId, database_id: crate::types::DatabaseId, + read_txn: Option, trace_id: TraceId, ) -> crate::Result> { crate::control::planner::implicit_edges::append_implicit_edge_tasks( @@ -181,7 +212,7 @@ pub async fn append_derived_tasks( trace_id, ) .await?; - append_sum_and_period_targets(state, tasks, tenant_id, database_id, trace_id).await + append_sum_and_period_targets(state, tasks, tenant_id, database_id, read_txn, trace_id).await } /// Resolve every write's materialized-sum targets, append one @@ -195,11 +226,16 @@ pub async fn append_derived_tasks( /// /// Returns the read-set entries covering the images every cross-shard /// balance was settled from. The caller unions them into its read set. +/// +/// `read_txn` is the open transaction the statement runs in, `None` outside a +/// transaction block. Each source read goes through that transaction's +/// staging overlay. pub async fn append_sum_and_period_targets( state: &SharedState, tasks: &mut Vec, tenant_id: crate::types::TenantId, database_id: crate::types::DatabaseId, + read_txn: Option, trace_id: TraceId, ) -> crate::Result> { let sum_target_reads = @@ -208,6 +244,7 @@ pub async fn append_sum_and_period_targets( tasks, tenant_id, database_id, + read_txn, trace_id, ) .await?; @@ -229,6 +266,7 @@ pub async fn append_sum_and_period_targets( tasks, tenant_id, database_id, + read_txn, trace_id, ) .await?; diff --git a/nodedb/src/control/server/shared/session/expander_stage.rs b/nodedb/src/control/server/shared/session/expander_stage.rs index bce388db5..7927ae991 100644 --- a/nodedb/src/control/server/shared/session/expander_stage.rs +++ b/nodedb/src/control/server/shared/session/expander_stage.rs @@ -122,12 +122,15 @@ pub(crate) async fn expand_in_tx( // The expanded point writes never passed statement admission. They take // its sum and period-lock pass here. A cross-shard balance ships on an // `ApplyBalanceDelta` task homed on the target's vShard, and the source - // write does not fold it. + // write does not fold it. The pre-image each balance settles from is read + // through the transaction's staging overlay, the same view the expansion + // resolved its rows against. let reads = append_sum_and_period_targets( state, &mut ops, task.tenant_id, task.database_id, + sessions.tx_id(session_id), crate::types::TraceId::ZERO, ) .await diff --git a/nodedb/src/control/server/shared/session/read_set.rs b/nodedb/src/control/server/shared/session/read_set.rs index 392958c5a..01b0e4093 100644 --- a/nodedb/src/control/server/shared/session/read_set.rs +++ b/nodedb/src/control/server/shared/session/read_set.rs @@ -89,16 +89,19 @@ pub enum ReadOrigin { /// began. Its observation can legitimately be superseded by the /// transaction's own writes, so the own-write exclusion applies. Session, - /// A read the Control Plane performed BEFORE the transaction existed, whose - /// observed value a value this transaction writes was computed from — a - /// materialized-sum settlement's pre-image is the case that exists today. + /// A read the Control Plane performed at plan time, whose observed value a + /// value this transaction writes was computed from — a materialized-sum + /// settlement's pre-image is the case that exists today. /// - /// This is NOT a read-your-own-write. It observed COMMITTED base state at a - /// point in time, and the derived value the transaction ships is only - /// correct if that observation still holds at apply time. Dropping it - /// because the transaction happens to write the same collection will - /// discard the one check that catches a concurrent writer moving the base - /// row out from under the derivation, so it survives the exclusion. + /// This is NOT a read-your-own-write. Its version is the COMMITTED base + /// state at a point in time, and the derived value the transaction ships is + /// only correct if that observation still holds at apply time. Inside a + /// transaction the read also folds the transaction's staging overlay, but + /// a staged write moves no committed version, so the check still compares + /// committed base only. Dropping it because the transaction happens to + /// write the same collection will discard the one check that catches a + /// concurrent writer moving the base row out from under the derivation, so + /// it survives the exclusion. PlanDerivation, } diff --git a/nodedb/src/control/server/shared/statement_exec/answer.rs b/nodedb/src/control/server/shared/statement_exec/answer.rs new file mode 100644 index 000000000..e7389f02d --- /dev/null +++ b/nodedb/src/control/server/shared/statement_exec/answer.rs @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! One statement's answer, folded across its tasks: its row sets, its +//! warnings and its one command tag. +//! +//! The fold is protocol-neutral. pgwire renders the same tag fold, native +//! renders a [`StatementAnswer`] as one response frame, and HTTP renders it as +//! JSON rows. + +use crate::bridge::envelope::PhysicalPlan; +use crate::control::server::response_shape::compose::{ShapeOutcome, shape_response_materialized}; +use crate::control::server::response_shape::redaction::QueryRedaction; +use crate::control::server::response_shape::request::MaterializedShapeRequest; +use crate::control::server::response_shape::types::{ + DmlOutcome, FoldedTag, PlanKind, ShapedRows, StatementTag, TaskTagRole, describe_plan, + payload_to_dml_outcome, staged_dml_outcome, +}; +use crate::control::server::shared::session::staging_gate::{StagedTagKind, StagedWriteOutcome}; +use nodedb_physical::physical_task::PhysicalTask; + +use super::error::StatementError; +use super::task_loop::StatementExec; + +/// A statement's finished answer. +pub(crate) struct StatementAnswer { + /// Each task's shaped rows, in dispatch order. A row set's notice is + /// already in `warnings`. + pub(crate) rows: Vec, + pub(crate) warnings: Vec, + /// The statement's one tag. `None` when no task folded into it. + pub(crate) tag: Option, + /// The watermark of the last task the statement dispatched. + pub(crate) last_lsn: u64, +} + +/// One statement's answer, accumulated across its tasks. +pub(crate) struct StatementFold { + rows: Vec, + warnings: Vec, + pub(crate) tag: StatementTag, + pub(crate) last_lsn: u64, +} + +impl StatementFold { + /// An empty fold for the statement that planned `tasks`. The tag is built + /// over every task, so a derived write beside the user's own adds no + /// verb and no count. + pub(crate) fn for_tasks(tasks: &[PhysicalTask]) -> Self { + Self { + rows: Vec::new(), + warnings: Vec::new(), + tag: StatementTag::for_plans(tasks.iter().map(|t| &t.plan)), + last_lsn: 0, + } + } + + /// Fold the count-bearing outcome of the task that ran `plan` into the + /// statement's tag. + pub(crate) fn fold_dml( + &mut self, + plan: &PhysicalPlan, + outcome: DmlOutcome, + ) -> Result<(), StatementError> { + self.tag + .fold(self.tag.role_of(plan), outcome) + .map_err(StatementError::DmlFold) + } + + /// Fold one shaped row set. Returns its row count. + pub(crate) fn fold_shaped(&mut self, mut shaped: ShapedRows) -> u64 { + if let Some(notice) = shaped.notice.take() { + self.warnings.push(notice); + } + let count = shaped.rows.len() as u64; + self.rows.push(shaped); + count + } + + /// Shape `payload`, the answer `plan` gave as `plan_kind`, and fold its + /// rows. Returns the row count, `None` when the payload carries no rows. + pub(crate) fn fold_rows( + &mut self, + exec: &StatementExec<'_>, + plan: &PhysicalPlan, + plan_kind: PlanKind, + payload: &[u8], + ) -> Result, StatementError> { + let redaction = QueryRedaction::for_plan(exec.tenant_id(), exec.scope.auth(), plan); + match shape_response_materialized(MaterializedShapeRequest { + payload, + plan, + plan_kind, + projection: exec.output_schema, + state: exec.state, + database_id: exec.database_id, + tenant_id: exec.tenant_id(), + redaction: Some(redaction.ctx(&exec.state.redaction)), + sequences: Some(exec.sequences), + }) { + Ok(ShapeOutcome::Rows(shaped)) => Ok(Some(self.fold_shaped(shaped))), + Ok(ShapeOutcome::Passthrough) => Ok(None), + Err(error) => Err(StatementError::Shape(error)), + } + } + + /// Fold a write its transaction staged. A write that answers `RETURNING` + /// folds its rows in place of a count, as an autocommit one does. A + /// computed-value payload (`RawPayload`) is shaped instead of counted. + pub(crate) fn fold_staged( + &mut self, + exec: &StatementExec<'_>, + plan: &PhysicalPlan, + outcome: StagedWriteOutcome, + ) -> Result<(), StatementError> { + if !outcome.returning_rows.is_empty() { + for rows in &outcome.returning_rows { + self.fold_rows(exec, plan, PlanKind::ReturningRows, rows)?; + } + return Ok(()); + } + if !matches!(outcome.kind, StagedTagKind::RawPayload) { + return self.fold_dml(plan, staged_dml_outcome(outcome.kind, outcome.affected)); + } + if outcome.payload.is_empty() { + self.tag.fold_opaque(); + return Ok(()); + } + // The value rides in the payload, never as a count. + if self + .fold_rows(exec, plan, describe_plan(plan), &outcome.payload)? + .is_none() + { + self.tag.fold_opaque(); + } + Ok(()) + } + + /// Fold a dispatched task's answer `payload`: its rows, or its count. + /// Returns the task's own row count for metering, `None` when it carries + /// none. + pub(crate) fn fold_answer( + &mut self, + exec: &StatementExec<'_>, + plan: &PhysicalPlan, + payload: &[u8], + ) -> Result, StatementError> { + let plan_kind = describe_plan(plan); + let count_bearing = matches!(plan_kind, PlanKind::DmlResult(_) | PlanKind::DmlResultByOp); + if payload.is_empty() && !count_bearing { + // Not a count-bearing plan (graph / vector / index write): no + // count and no verb to report. A count-bearing plan with no + // payload falls through so the count reader refuses it. + self.tag.fold_opaque(); + return Ok(None); + } + if let Some(rows) = self.fold_rows(exec, plan, plan_kind, payload)? { + return Ok(Some(rows)); + } + // A count-bearing write reports the rows it touched and its verb; an + // opaque execution reports neither. A count of one per dispatched + // task reports a row for a delete that removed nothing and for an + // `ON CONFLICT DO NOTHING` insert that skipped. + if self.tag.role_of(plan) == TaskTagRole::Opaque { + self.tag.fold_opaque(); + return Ok(None); + } + match payload_to_dml_outcome(payload, plan_kind) { + Ok(Some(outcome)) => { + let affected = outcome.affected; + self.fold_dml(plan, outcome)?; + Ok(Some(affected)) + } + Ok(None) => { + self.tag.fold_opaque(); + Ok(None) + } + Err(error) => Err(StatementError::Error(error)), + } + } + + /// The statement's answer. + pub(crate) fn finish(self) -> StatementAnswer { + StatementAnswer { + rows: self.rows, + warnings: self.warnings, + tag: self.tag.finish(), + last_lsn: self.last_lsn, + } + } +} diff --git a/nodedb/src/control/server/shared/statement_exec/atomic.rs b/nodedb/src/control/server/shared/statement_exec/atomic.rs new file mode 100644 index 000000000..1528899d7 --- /dev/null +++ b/nodedb/src/control/server/shared/statement_exec/atomic.rs @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The routes that commit a planned statement's tasks together, taken ahead +//! of its per-task loop. Native, HTTP, WebSocket RPC and NDJSON call +//! [`route_atomic_statement`] for a planned SQL statement, so the same +//! statement takes the same route over each of them, as it does over pgwire. +//! +//! In order: +//! +//! 1. A statement outside a transaction block that +//! [`statement_needs_implicit_txn`] flags runs in an implicit +//! transaction: its writes stage on their vShards' leaders and commit +//! together at its end, through Calvin when they span vShards. +//! 2. A dependent predicate on an edge-bearing collection runs through the +//! implicit-edge OLLP/Calvin coordinator. +//! 3. A write that spans vShards commits through the Calvin sequencer, so a +//! derived task (a cross-shard balance move, an implicit edge) commits +//! with the user's write. + +use std::sync::Arc; + +use nodedb_physical::physical_task::PhysicalTask; + +use crate::control::lease::QueryLeaseScope; +use crate::control::planner::calvin::{ + CrossShardTxnMode, DispatchClass, TxnDispatchPosition, classify_dispatch, + dispatch_authorized_tasks_to_calvin, read_vshards_of, +}; +use crate::control::security::audit::ArcAuditEmitter; +use crate::control::server::response_shape::calvin_fold::{ + CalvinFoldCtx, CalvinFoldError, fold_calvin_batch, +}; +use crate::control::server::shared::authorization::authorize_task_set; +use crate::control::server::shared::metering::{PlanMeteringInfo, meter_dispatch}; +use crate::control::server::shared::session::DmlTxnCtx; +use crate::control::server::shared::session::read_set::ReadSetEntry; +use crate::control::server::shared::txn_route::{ + statement_needs_implicit_txn, unbufferable_joined_statement, +}; +use crate::control::server::shared::write_admission::all_writes_bufferable; +use crate::control::trigger::statement_txn::in_block; + +use super::answer::StatementAnswer; +use super::edge_recon::{CalvinApplied, EdgeRecon, try_edge_recon}; +use super::error::StatementError; +use super::task_loop::{PlannedStatement, StatementExec, run_implicit_statement}; + +/// A planned statement before it takes a route. +pub(crate) struct AtomicStatement { + pub(crate) tasks: Vec, + /// The statement's descriptor leases. They stay alive through dispatch. + pub(crate) lease_scope: QueryLeaseScope, + /// The row images the statement's cross-shard balances were settled + /// from. They widen Calvin's read set, so its OCC check aborts a balance + /// folded from an image that has since moved. + pub(crate) sum_target_reads: Vec, +} + +/// The route a statement took. +pub(crate) enum AtomicRoute { + /// The statement ran in its implicit transaction. + Implicit(StatementAnswer), + /// Calvin applied the statement's tasks as one transaction. + Calvin(CalvinApplied), + /// No atomic route applies. The caller runs its per-task loop. + PerTask(AtomicStatement), +} + +/// Route `statement`, planned on `session`, through the first atomic route +/// that applies. A transport with no session passes a detached one, which is +/// never in a transaction block and takes the default cross-shard mode. +pub(crate) async fn route_atomic_statement( + exec: &StatementExec<'_>, + statement: AtomicStatement, + session: &DmlTxnCtx<'_>, +) -> Result { + let AtomicStatement { + tasks, + lease_scope, + sum_target_reads, + } = statement; + let state = exec.state; + + // Calvin dispatch and edge reconciliation are trusted internal mechanisms + // that never reach the clone-checked dispatch boundary, so one batch + // authorize is correct for them. The per-task loop authorizes each task + // itself, immediately before its own dispatch. + let authorized = authorize_task_set( + exec.identity, + &tasks, + &state.permissions, + &state.roles, + &ArcAuditEmitter(Arc::clone(&state.audit)), + ) + .map_err(|error| StatementError::Error(crate::Error::from(error)))?; + + let in_txn_block = in_block(session); + if !in_txn_block && statement_needs_implicit_txn(state, &tasks) { + if !all_writes_bufferable(&tasks) { + return Err(StatementError::Error(unbufferable_joined_statement())); + } + let answer = run_implicit_statement( + exec, + PlannedStatement { + tasks, + plan_lease_scope: Arc::new(lease_scope), + sum_target_reads, + }, + session, + ) + .await?; + return Ok(AtomicRoute::Implicit(answer)); + } + + let (tasks, authorized) = + match try_edge_recon(state, exec.identity, in_txn_block, tasks, authorized) + .await + .map_err(StatementError::Error)? + { + EdgeRecon::Applied(applied) => { + meter_applied(exec, &applied); + return Ok(AtomicRoute::Calvin(applied)); + } + EdgeRecon::NotFired(tasks, authorized) => (tasks, authorized), + }; + + // An autocommit statement has no session read set to widen with. The + // reads to widen with are those materialized-sum settlement stamped. + let sum_read_vshards = read_vshards_of(&sum_target_reads).map_err(StatementError::Error)?; + match classify_dispatch(&tasks, &sum_read_vshards) { + DispatchClass::SingleShard { .. } => {} + DispatchClass::MultiShard { .. } => { + // Calvin applies the statement durably at statement time, + // outside any transaction buffer. Inside a block the per-task + // loop buffers each write, and COMMIT flushes the buffer through + // Calvin. A write the gate cannot buffer is refused. + if in_txn_block && !all_writes_bufferable(&tasks) { + return Err(StatementError::Error( + crate::Error::CrossShardInExplicitTransaction, + )); + } + let cross_shard_mode = session.sessions.cross_shard_txn_mode(session.session_id); + if !in_txn_block && cross_shard_mode == CrossShardTxnMode::Strict { + let plans = tasks.iter().map(|task| task.plan.clone()).collect(); + let apply_result = dispatch_authorized_tasks_to_calvin( + state, + authorized, + exec.tenant_id(), + cross_shard_mode, + TxnDispatchPosition::Autocommit, + &sum_target_reads, + None, + ) + .await + .map_err(StatementError::Error)?; + let applied = CalvinApplied { + apply_result, + plans, + database_id: exec.database_id, + }; + meter_applied(exec, &applied); + return Ok(AtomicRoute::Calvin(applied)); + } + // `BestEffortNonAtomic` falls through to the per-task loop. + } + } + + Ok(AtomicRoute::PerTask(AtomicStatement { + tasks, + lease_scope, + sum_target_reads, + })) +} + +/// Fold a statement Calvin applied into its answer: the RETURNING rows, when +/// a task carried them, and its one tag. The fold is the one pgwire renders. +pub(crate) fn calvin_answer( + exec: &StatementExec<'_>, + applied: CalvinApplied, +) -> Result { + let CalvinApplied { + apply_result, + plans, + database_id, + } = applied; + let plan_refs: Vec<_> = plans.iter().collect(); + let fold = fold_calvin_batch( + &plan_refs, + apply_result.as_ref(), + &CalvinFoldCtx { + projection: exec.output_schema, + state: exec.state, + tenant_id: exec.tenant_id(), + database_id, + auth: exec.scope.auth(), + }, + ) + .map_err(|error| match error { + CalvinFoldError::Verb(error) => StatementError::DmlFold(error), + CalvinFoldError::Shape(error) => StatementError::Error(error), + })?; + let mut warnings = Vec::new(); + let rows = match fold.rows { + Some(mut shaped) => { + if let Some(notice) = shaped.notice.take() { + warnings.push(notice); + } + vec![shaped] + } + None => Vec::new(), + }; + Ok(StatementAnswer { + rows, + warnings, + tag: fold.tag, + last_lsn: apply_result.map_or(0, |response| response.watermark_lsn.as_u64()), + }) +} + +/// Meter each task of a statement Calvin applied, as pgwire does. +fn meter_applied(exec: &StatementExec<'_>, applied: &CalvinApplied) { + if !exec.state.metering_config.enabled { + return; + } + for plan in &applied.plans { + meter_dispatch( + exec.state, + exec.scope, + &PlanMeteringInfo::extract(plan), + None, + ); + } +} diff --git a/nodedb/src/control/server/shared/statement_exec/dispatch.rs b/nodedb/src/control/server/shared/statement_exec/dispatch.rs new file mode 100644 index 000000000..771704626 --- /dev/null +++ b/nodedb/src/control/server/shared/statement_exec/dispatch.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The single-task dispatch primitive a planned statement's loop calls. +//! +//! Decides HOW one already-planned task reaches an engine: a Control-Plane +//! orchestrator, Exchange resolution, then the gateway. The statement loop +//! decides what to do with the answers. + +use std::sync::Arc; + +use nodedb_physical::physical_plan::DocumentOp; +use nodedb_physical::physical_task::PhysicalTask; + +use crate::bridge::envelope::{PhysicalPlan, Response}; +use crate::control::gateway::core::QueryContext as GatewayQueryContext; +use crate::control::gateway::router::is_task_vshard_scoped; +use crate::control::security::audit::ArcAuditEmitter; +use crate::control::security::identity::AuthenticatedIdentity; +use crate::control::server::exchange::resolve::{Resolved, resolve_and_materialize}; +use crate::control::server::exchange::{DistributedReadCapture, ReadScope}; +use crate::control::server::shared::authorization::{AuthorizedTask, authorize_task_set}; +use crate::control::server::shared::clone_write::{ + CloneCheckedOutcome, InterceptAndAuthorizeParams, intercept_and_authorize, +}; +use crate::control::state::SharedState; +use crate::types::{Lsn, TraceId, VShardId}; + +/// One dispatched task's answer. +pub(crate) struct DispatchedTask { + pub(crate) response: Response, + /// The watermark of each shard a single-node fan gather read. + pub(crate) shard_watermarks: Vec<(VShardId, Lsn)>, + /// The reads a distributed gather captured. + pub(crate) distributed_reads: Vec, +} + +impl DispatchedTask { + fn answered(response: Response) -> Self { + Self { + response, + shard_watermarks: Vec::new(), + distributed_reads: Vec::new(), + } + } +} + +/// Authorize one task with no clone-write check. Used only for the +/// Control-Plane orchestrated plans (`InsertSelect`, `Merge`, +/// `UpdateFromJoin`, a governed predicate resolution, array DDL, a cluster +/// array op), which are never clone-write shapes. +pub(crate) fn authorize_one_task( + state: &SharedState, + identity: &AuthenticatedIdentity, + task: &PhysicalTask, +) -> crate::Result { + let emitter = ArcAuditEmitter(Arc::clone(&state.audit)); + authorize_task_set( + identity, + std::slice::from_ref(task), + &state.permissions, + &state.roles, + &emitter, + ) + .map_err(crate::Error::from)? + .into_tasks() + .into_iter() + .next() + .ok_or_else(|| crate::Error::Internal { + detail: "authorization returned an empty capability set".into(), + }) +} + +/// Dispatch one `PhysicalTask` and return its answer. +/// +/// Multi-arm writes run through Control-Plane orchestrators. Everything else +/// resolves its Exchange nodes, then routes through +/// [`dispatch_task_via_gateway`]. +pub(crate) async fn dispatch_statement_task( + state: &Arc, + identity: &AuthenticatedIdentity, + mut task: PhysicalTask, +) -> crate::Result { + if let PhysicalPlan::Document(DocumentOp::InsertSelect { .. }) = &task.plan { + let authorized = authorize_one_task(state, identity, &task)?; + let response = + crate::control::insert_select::run_authorized_insert_select(state, authorized).await?; + return Ok(DispatchedTask::answered(response)); + } + + // Autocommit `MERGE` orchestrates on the Control Plane. + if let PhysicalPlan::Document(DocumentOp::Merge { + target_collection: _, + source_collection: _, + source_alias: _, + target_join_col: _, + source_join_col: _, + clauses: _, + returning: _, + resolved_inserts: None, + resolved_insert_identities: _, + source_rows: _, + rls_filters: _, + rls_write_check: _, + resolved_sum_targets: _, + declared_primary_key: _, + }) = &task.plan + { + let authorized = authorize_one_task(state, identity, &task)?; + let response = + crate::control::merge_orchestrator::run_authorized_merge(state, authorized).await?; + return Ok(DispatchedTask::answered(response)); + } + + // Autocommit `UPDATE ... FROM ` scans the source on its own core + // and ships it into the plan, since the source's vShard can live on a + // different core. + if let PhysicalPlan::Document(DocumentOp::UpdateFromJoin { + target_collection: _, + source_collection: _, + source_alias: _, + target_join_col: _, + source_join_col: _, + updates: _, + target_filters: _, + returning: _, + source_rows: None, + rls_filters: _, + rls_write_check: _, + resolved_sum_targets: _, + declared_primary_key: _, + }) = &task.plan + { + let authorized = authorize_one_task(state, identity, &task)?; + let response = + crate::control::update_from_join_orchestrator::run_authorized_update_from_join( + state, authorized, + ) + .await?; + return Ok(DispatchedTask::answered(response)); + } + + // A governed predicate resolves to a concrete row set before proposing. + if let Some(resolver) = crate::control::write_resolve::resolver_for_plan(&task.plan) { + let authorized = authorize_one_task(state, identity, &task)?; + let response = crate::control::write_resolve::run_authorized_write_resolve( + state, authorized, resolver, + ) + .await?; + return Ok(DispatchedTask::answered(response)); + } + + // Array DDL proposes a replicated catalog entry. + if crate::control::array_catalog::ddl::is_array_ddl(&task.plan) { + let authorized = authorize_one_task(state, identity, &task)?; + let response = + crate::control::array_catalog::ddl::run_authorized_array_ddl(state, authorized).await?; + return Ok(DispatchedTask::answered(response)); + } + + // Materialize catalog providers and resolve Exchange nodes before + // dispatch. Every statement read is strong, so every leg confirms its + // group where it is served. + let scope = ReadScope { + database_id: task.database_id, + tenant_id: task.tenant_id, + trace_id: TraceId::ZERO, + txn_id: task.txn_id, + linearizable: true, + }; + match resolve_and_materialize(state, identity, task.plan, scope).await? { + Resolved::Gathered(response, shard_watermarks, distributed_reads) => { + return Ok(DispatchedTask { + response, + shard_watermarks, + distributed_reads, + }); + } + Resolved::Plan(resolved_plan) => { + task.plan = *resolved_plan; + } + // A materialized statement gathers the stream into one response. + Resolved::Stream(stream) => { + let response = + crate::control::server::exchange::gather::stream_to_response(stream).await?; + return Ok(DispatchedTask::answered(response)); + } + } + + let response = dispatch_task_via_gateway(state, identity, task).await?; + Ok(DispatchedTask::answered(response)) +} + +/// Dispatch one `PhysicalTask` through the gateway. +/// +/// The task takes the clone copy-on-write gate and authorization first. A +/// staged write and the other transaction meta-ops run on the core of the +/// task's own vShard. Every other task routes through +/// `Gateway::execute_response`, which handles cluster-aware routing, typed +/// `NotLeader` retry and plan caching. A `NotFound` verdict comes back as an +/// error status. +pub(crate) async fn dispatch_task_via_gateway( + state: &Arc, + identity: &AuthenticatedIdentity, + task: PhysicalTask, +) -> crate::Result { + let tenant_id = task.tenant_id; + let emitter = ArcAuditEmitter(Arc::clone(&state.audit)); + let checked = match intercept_and_authorize(InterceptAndAuthorizeParams { + state, + task, + identity, + tenant_id, + permissions: &state.permissions, + roles: &state.roles, + emitter: &emitter, + }) + .await? + { + CloneCheckedOutcome::Handled(response) => return Ok(response), + CloneCheckedOutcome::Proceed(checked) => checked, + }; + let database_id = checked.database_id(); + let txn_id = checked.txn_id(); + + let gateway = state.installed_gateway()?; + // The gateway routes a vShard-scoped meta-op to vShard 0. A write takes + // the durable route, a read the read route. + if is_task_vshard_scoped(checked.plan()) { + return crate::control::server::dispatch_utils::dispatch_authorized_task_by_class( + state, + checked, + TraceId::generate(), + ) + .await; + } + let gw_ctx = GatewayQueryContext { + tenant_id, + trace_id: TraceId::generate(), + database_id, + // The in-block transaction id lets local dispatch resolve the + // per-transaction staging overlay. + txn_id, + linearizable: true, + }; + gateway.execute_response(&gw_ctx, checked).await +} diff --git a/nodedb/src/control/server/shared/statement_exec/edge_recon.rs b/nodedb/src/control/server/shared/statement_exec/edge_recon.rs new file mode 100644 index 000000000..21a0ffb11 --- /dev/null +++ b/nodedb/src/control/server/shared/statement_exec/edge_recon.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The implicit-edge OLLP/Calvin gate. +//! +//! A dependent predicate (`BulkDelete`/`BulkUpdate`) on an edge-bearing +//! schemaless collection routes through the OLLP/Calvin coordinator, so the +//! implicit edge tasks derive from a pre-exec reconnaissance scan and commit +//! atomically with the document write. Without this gate such a delete +//! classifies as `SingleShard`, dispatches directly to the Data Plane, and +//! leaves its mirrored CSR edges dangling. + +use std::sync::Arc; + +use nodedb_physical::physical_task::PhysicalTask; + +use crate::bridge::envelope::{PhysicalPlan, Response}; +use crate::control::planner::calvin::{ + dispatch_authorized_dependent_edge_recon, plan_needs_implicit_edge_recon, +}; +use crate::control::security::identity::AuthenticatedIdentity; +use crate::control::server::shared::authorization::AuthorizedTaskSet; +use crate::control::state::SharedState; +use crate::types::{DatabaseId, TenantId}; + +/// A statement Calvin applied as one transaction, before any protocol +/// renders its answer. +pub(crate) struct CalvinApplied { + /// The applied response Calvin deposited, when a task deposits one. + pub(crate) apply_result: Option, + /// Every task's plan, in dispatch order. + pub(crate) plans: Vec, + /// The database the answer shapes against. + pub(crate) database_id: DatabaseId, +} + +/// What the gate did with a statement. +pub(crate) enum EdgeRecon { + /// The gate did not fire. The caller gets the tasks back. + NotFired(Vec, AuthorizedTaskSet), + /// The coordinator applied the statement. + Applied(CalvinApplied), +} + +/// Route `tasks` through the implicit-edge OLLP/Calvin coordinator when one +/// of them is a `BulkDelete`/`BulkUpdate` on an edge-bearing collection and +/// the statement runs outside a transaction block. +/// +/// Inside a block the gate does not fire: a multi-step OLLP inside the +/// client's transaction needs a two-phase commit across its boundary. A +/// catalog I/O error propagates. Misrouting on a real fault skips the edge +/// cleanup. +pub(crate) async fn try_edge_recon( + state: &Arc, + identity: &AuthenticatedIdentity, + in_txn_block: bool, + tasks: Vec, + authorized: AuthorizedTaskSet, +) -> crate::Result { + if in_txn_block { + return Ok(EdgeRecon::NotFired(tasks, authorized)); + } + let tenant_id: TenantId = identity.tenant_id; + let Some((_collection, database_id)) = + plan_needs_implicit_edge_recon(state, &tasks, tenant_id)? + else { + return Ok(EdgeRecon::NotFired(tasks, authorized)); + }; + // Captured before the coordinator consumes the tasks, so the answer can + // shape a RETURNING dependent write's rows and read a plain write's + // count from the applied response. + let plans: Vec = tasks.iter().map(|task| task.plan.clone()).collect(); + let recon = dispatch_authorized_dependent_edge_recon( + state, + authorized, + identity, + tenant_id, + database_id, + ) + .await?; + Ok(EdgeRecon::Applied(CalvinApplied { + apply_result: recon.apply_result, + plans, + database_id, + })) +} diff --git a/nodedb/src/control/server/shared/statement_exec/error.rs b/nodedb/src/control/server/shared/statement_exec/error.rs new file mode 100644 index 000000000..be11b01b6 --- /dev/null +++ b/nodedb/src/control/server/shared/statement_exec/error.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Why a planned statement stopped, before any protocol renders it. + +use crate::bridge::envelope::{ErrorCode, Response}; +use crate::control::server::response_shape::types::DmlFoldError; +use crate::types::TenantId; + +/// Why a statement stopped. Each protocol renders it in its own error shape. +pub(crate) enum StatementError { + /// A Control-Plane error. + Error(crate::Error), + /// A spent hard quota refused a task before it ran. + Quota(crate::Error), + /// A task names a tenant other than the statement's. + TenantIsolation, + /// The Data Plane refused a staged write. `None` when it gave no code. + Rejected(Option), + /// The Data Plane answered a dispatched task with an error status. + Response(Box), + /// Two tasks of the statement report verbs that cannot share one tag. + DmlFold(DmlFoldError), + /// A task's answer cannot be shaped. + Shape(nodedb_types::NodeDbError), +} + +impl StatementError { + /// The Control-Plane error a protocol with no finer error shape renders. + /// `tenant_id` is the statement's tenant. + pub(crate) fn into_error(self, tenant_id: TenantId) -> crate::Error { + match self { + Self::Error(error) | Self::Quota(error) => error, + Self::TenantIsolation => crate::Error::RejectedAuthz { + tenant_id, + resource: "tenant isolation violation".into(), + }, + Self::Rejected(Some(code)) => crate::Error::DataPlane(code), + Self::Rejected(None) => crate::Error::Internal { + detail: "unknown data plane error".into(), + }, + Self::Response(response) => match response.error_code { + Some(code) => crate::Error::DataPlane(*code), + None => crate::Error::Internal { + detail: "data plane returned an error status with no error code".into(), + }, + }, + Self::DmlFold(error) => crate::Error::Internal { + detail: error.to_string(), + }, + Self::Shape(error) => crate::Error::from(error), + } + } +} diff --git a/nodedb/src/control/server/shared/statement_exec/mod.rs b/nodedb/src/control/server/shared/statement_exec/mod.rs new file mode 100644 index 000000000..52b0928a6 --- /dev/null +++ b/nodedb/src/control/server/shared/statement_exec/mod.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! One planned SQL statement, run the same way by every protocol: its atomic +//! routes, its per-task loop, the single-task dispatch primitive, and the +//! fold of its answer. Each protocol renders the answer in its own shape. + +pub mod answer; +pub mod atomic; +pub mod dispatch; +pub mod edge_recon; +pub mod error; +pub mod task_loop; + +pub(crate) use answer::StatementAnswer; +pub(crate) use atomic::{AtomicRoute, AtomicStatement, calvin_answer, route_atomic_statement}; +pub(crate) use dispatch::authorize_one_task; +pub(crate) use edge_recon::{EdgeRecon, try_edge_recon}; +pub(crate) use error::StatementError; +pub(crate) use task_loop::{ + PlannedStatement, StatementExec, run_implicit_statement, run_statement_loop, +}; diff --git a/nodedb/src/control/server/shared/statement_exec/task_loop.rs b/nodedb/src/control/server/shared/statement_exec/task_loop.rs new file mode 100644 index 000000000..cda8c921d --- /dev/null +++ b/nodedb/src/control/server/shared/statement_exec/task_loop.rs @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The per-task loop of a planned statement. +//! +//! A statement in a transaction (the client's block, or the implicit +//! transaction a statement runs in when [`statement_needs_implicit_txn`] +//! flags it) routes each task through the shared `txn_route`, the route +//! pgwire takes too: the write's trigger bodies join the transaction, a +//! shadowed clone takes its copy-on-write steps, and the write stages. An +//! autocommit statement dispatches each task directly. +//! +//! [`statement_needs_implicit_txn`]: crate::control::server::shared::txn_route::statement_needs_implicit_txn + +use std::sync::Arc; + +use nodedb_physical::physical_task::PhysicalTask; + +use crate::bridge::envelope::{ErrorCode, PhysicalPlan, Status}; +use crate::control::lease::QueryLeaseScope; +use crate::control::security::identity::AuthenticatedIdentity; +use crate::control::security::request_scope::RequestAuthScope; +use crate::control::sequence::SessionSequenceAccess; +use crate::control::server::response_shape::schema::OutputSchema; +use crate::control::server::response_shape::types::{describe_plan, replaced_write_outcome}; +use crate::control::server::shared::cluster_array_dispatch::{ + ClusterArrayShaped, execute_cluster_array, +}; +use crate::control::server::shared::metering::{PlanMeteringInfo, meter_dispatch}; +use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; +use crate::control::server::shared::session::read_set::ReadSetEntry; +use crate::control::server::shared::session::staging_gate::StagingGateError; +use crate::control::server::shared::session::{ + DmlTxnCtx, ResponseReads, record_reads_for_response, +}; +use crate::control::server::shared::txn_route::{ + StatementEvents, TxnTaskContext, TxnTaskOutcome, route_txn_task, +}; +use crate::control::state::SharedState; +use crate::types::{DatabaseId, TenantId}; + +use super::answer::{StatementAnswer, StatementFold}; +use super::dispatch::{DispatchedTask, authorize_one_task, dispatch_statement_task}; +use super::error::StatementError; + +/// Who runs a statement, and what its answer shapes against. +pub(crate) struct StatementExec<'a> { + pub(crate) state: &'a Arc, + pub(crate) identity: &'a AuthenticatedIdentity, + /// The request's auth scope: RLS context, redaction, metering and quota. + pub(crate) scope: &'a RequestAuthScope<'a>, + /// The statement's announced output columns, when it announced any. + pub(crate) output_schema: Option<&'a OutputSchema>, + pub(crate) database_id: DatabaseId, + /// The sequence access the statement's Control-Plane computed columns + /// read: the registry, plus the session's `currval` map when it has one. + pub(crate) sequences: &'a SessionSequenceAccess<'a>, +} + +impl StatementExec<'_> { + pub(crate) fn tenant_id(&self) -> TenantId { + self.identity.tenant_id + } +} + +/// A planned statement's tasks and what they carry into dispatch. +pub(crate) struct PlannedStatement { + pub(crate) tasks: Vec, + /// The statement's descriptor leases, kept on every task it buffers. + pub(crate) plan_lease_scope: Arc, + /// The row images the statement's cross-shard balances were settled + /// from. A statement in a transaction adds them to its read set. + pub(crate) sum_target_reads: Vec, +} + +/// Run every task of `statement` and fold their answers into one. +/// `txn` is the statement's transaction, `None` for an autocommit statement. +/// +/// The answer's tag is one [`crate::control::server::response_shape::types::StatementTag`] +/// folded over every task, the same fold pgwire renders as its command tag: +/// a count-bearing task contributes its reported count and verb, an opaque +/// task (buffered write, index or graph maintenance, computed-value payload, +/// derived balance or edge write) contributes nothing. Neither is ever +/// synthesised from the task count. +pub(crate) async fn run_statement_loop( + exec: &StatementExec<'_>, + statement: PlannedStatement, + txn: Option<&DmlTxnCtx<'_>>, +) -> Result { + let PlannedStatement { + tasks, + plan_lease_scope, + sum_target_reads, + } = statement; + if let Some(txn) = txn + && !sum_target_reads.is_empty() + { + txn.sessions + .record_read_entries(txn.session_id, sum_target_reads); + } + let state = exec.state; + let mut fold = StatementFold::for_tasks(&tasks); + // Checked once rather than per task: metering is disabled by default, so + // the per-task extraction below stays a no-op on the hot path. + let metering_enabled = state.metering_config.enabled; + let route_ctx = txn.map(|txn| TxnTaskContext { + state, + identity: exec.identity, + auth: exec.scope.auth(), + txn, + lease_scope: &plan_lease_scope, + fire_triggers: true, + }); + let mut statement_events = StatementEvents::default(); + + for task in tasks { + if task.tenant_id != exec.tenant_id() { + return Err(StatementError::TenantIsolation); + } + + // Cloned before routing consumes `task`: a staged write's rows and + // computed-value payload shape against it. + let plan = task.plan.clone(); + // Metering covers the direct dispatch below. A staged write meters + // itself inside the staging gate, and a buffered write meters at + // COMMIT. + let plan_metering_info = metering_enabled.then(|| PlanMeteringInfo::extract(&plan)); + + // A spent hard quota refuses the task before it runs. The charging + // call at the end of this loop is on the success path and never + // refuses. + if let Some(info) = &plan_metering_info { + admit_quota_for_dispatch(state, exec.scope, info).map_err(StatementError::Quota)?; + } + + // The task to dispatch below: the input task, or the one routing + // hands back. Every other routing outcome answers the task here. + let task = match route_ctx.as_ref() { + Some(route) => { + let routed = route_txn_task( + route, + task, + &mut statement_events, + |stage_task| async move { + dispatch_statement_task(state, exec.identity, stage_task) + .await + .map(|dispatched| dispatched.response) + }, + ) + .await; + match routed { + Ok(TxnTaskOutcome::Dispatch(routed_task)) => *routed_task, + Ok(TxnTaskOutcome::Buffered) => { + // Applied at COMMIT: no count and no verb yet. + fold.tag.fold_opaque(); + continue; + } + Ok(TxnTaskOutcome::Staged(outcome)) => { + fold.fold_staged(exec, &plan, outcome)?; + continue; + } + Ok(TxnTaskOutcome::CloneHandled(clone_response)) => { + fold.fold_answer(exec, &plan, clone_response.payload.as_ref())?; + continue; + } + Ok(TxnTaskOutcome::InsteadOf) => { + match replaced_write_outcome(describe_plan(&plan)) { + Some(outcome) => fold.fold_dml(&plan, outcome)?, + None => fold.tag.fold_opaque(), + } + continue; + } + Err(StagingGateError::Dispatch(error)) => { + return Err(StatementError::Error(error)); + } + Err(StagingGateError::Rejected { code }) => { + return Err(StatementError::Rejected(code)); + } + } + } + None => task, + }; + + // `ClusterArray` plans run entirely on the Control Plane through the + // `ArrayCoordinator`, never through the SPSC bridge or the trigger/DML + // machinery. A write in a transaction reshapes into per-shard + // `ArrayOp` tasks inside the staging gate above, so only reads + // (`Slice`/`Agg`) and autocommit `Put`/`Delete` arrive here. Metering + // is not applied here, matching pgwire's `ClusterArray` short-circuit. + if matches!(task.plan, PhysicalPlan::ClusterArray(_)) { + let authorized = + authorize_one_task(state, exec.identity, &task).map_err(StatementError::Error)?; + match execute_cluster_array(state, exec.scope.auth(), authorized, exec.output_schema) + .await + .map_err(StatementError::Error)? + { + ClusterArrayShaped::Rows(shaped) => { + fold.fold_shaped(shaped); + } + ClusterArrayShaped::Affected(outcome) => fold.fold_dml(&plan, outcome)?, + } + continue; + } + + let task_vshard = task.vshard_id; + let task_database_id = task.database_id; + let DispatchedTask { + response: task_response, + shard_watermarks, + distributed_reads, + } = dispatch_statement_task(state, exec.identity, task) + .await + .map_err(StatementError::Error)?; + + // Track reads for snapshot-isolation / cross-shard conflict detection + // into the statement's transaction. Recorded BEFORE the error + // short-circuit so an absent-key point read (a `NotFound` from the + // Data Plane) is captured too: a "not found" is a validatable phantom + // observation. A multi-core fan read records one entry per + // participating shard from the gather's per-shard watermarks. A + // single read falls back to its one watermark. + let records_read = task_response.status == Status::Ok + || task_response.error_code.as_deref() == Some(&ErrorCode::NotFound); + if records_read && let Some(txn) = txn { + let watermarks = if shard_watermarks.is_empty() { + vec![(task_vshard, task_response.watermark_lsn)] + } else { + shard_watermarks + }; + record_reads_for_response( + state, + txn.sessions, + txn.session_id, + exec.tenant_id(), + ResponseReads { + plan: &plan, + watermarks: &watermarks, + read_version_lsn: task_response.read_version_lsn, + found: task_response.status == Status::Ok, + distributed_reads: &distributed_reads, + read_lsn_vshard: task_vshard, + }, + ) + .await; + } + + if task_response.status == Status::Error { + return Err(StatementError::Response(Box::new(task_response))); + } + + // TRUNCATE RESTART IDENTITY. Autocommit only: a buffered truncate + // restarts its sequences at COMMIT. + if let Some((collection, true)) = plan.truncate_target() { + state.sequence_registry.restart_sequences_for_collection( + task_database_id.as_u64(), + exec.tenant_id().as_u64(), + collection.as_str(), + ); + } + + fold.last_lsn = task_response.watermark_lsn.as_u64(); + let task_rows = fold.fold_answer(exec, &plan, task_response.payload.as_ref())?; + + // Metered once per successfully dispatched task. + if let Some(info) = &plan_metering_info { + meter_dispatch(state, exec.scope, info, task_rows); + } + } + + if let Some(route) = route_ctx.as_ref() { + statement_events + .fire(route) + .await + .map_err(StatementError::Error)?; + } + + Ok(fold.finish()) +} + +/// Run a statement in an implicit transaction: it commits when every task +/// succeeds, and rolls back when one fails. `client` is the session the +/// statement arrived on. The transaction runs on a private session of its +/// own, so a transport with no session passes a detached one. +pub(crate) async fn run_implicit_statement( + exec: &StatementExec<'_>, + statement: PlannedStatement, + client: &DmlTxnCtx<'_>, +) -> Result { + crate::control::trigger::statement_txn::with_statement_txn_lifted( + exec.state, + exec.identity, + client, + true, + StatementError::Error, + async |txn: &DmlTxnCtx<'_>| run_statement_loop(exec, statement, Some(txn)).await, + ) + .await +} diff --git a/nodedb/src/control/system_txn/tests.rs b/nodedb/src/control/system_txn/tests.rs index f7e8962c9..363887b31 100644 --- a/nodedb/src/control/system_txn/tests.rs +++ b/nodedb/src/control/system_txn/tests.rs @@ -214,6 +214,7 @@ async fn a_body_plan_carries_only_stageable_forms() { &mut tasks, TenantId::new(1), DatabaseId::DEFAULT, + None, crate::types::TraceId::ZERO, ) .await diff --git a/nodedb/src/control/update_from_join_orchestrator/orchestrator.rs b/nodedb/src/control/update_from_join_orchestrator/orchestrator.rs index 951d6baa4..7bb6b0e50 100644 --- a/nodedb/src/control/update_from_join_orchestrator/orchestrator.rs +++ b/nodedb/src/control/update_from_join_orchestrator/orchestrator.rs @@ -105,15 +105,6 @@ pub(crate) async fn run_update_from_join( state: &SharedState, args: UpdateFromJoinArgs<'_>, ) -> crate::Result { - // The write applies on the target's vShard alone, so it folds only - // co-resident balances. - crate::control::planner::materialized_sum::refuse_cross_shard_orchestration( - state, - args.target_collection, - args.tenant_id, - args.database_id, - "UPDATE ... FROM", - )?; // Checked once: a target driving no materialized-sum binding, declaring // no period lock AND carrying no write policy skips the RESOLVE round // trip and retry loop entirely. A write policy needs the matched diff --git a/nodedb/src/data/executor/enforcement/materialized_sum/apply.rs b/nodedb/src/data/executor/enforcement/materialized_sum/apply.rs index 6304360a2..d36f30657 100644 --- a/nodedb/src/data/executor/enforcement/materialized_sum/apply.rs +++ b/nodedb/src/data/executor/enforcement/materialized_sum/apply.rs @@ -45,10 +45,10 @@ //! //! No write path hands this core a cross-shard target it must apply. Plain //! writes and the in-transaction expanders take the same Control-Plane pass, -//! which ships every cross-shard balance on its own task. The autocommit -//! orchestrators (`MERGE`, `UPDATE ... FROM`, `INSERT ... SELECT`) refuse a -//! source that drives a cross-shard target, and the SQL and native protocols -//! run such a statement through the expanders instead. +//! which ships every cross-shard balance on its own task. A `MERGE`, +//! `UPDATE ... FROM` or `INSERT ... SELECT` into a source that drives a +//! cross-shard target never reaches the autocommit orchestrators: every +//! protocol runs it through the expanders in an implicit transaction. //! //! # Identity comes from the plan, never from a store probe //! diff --git a/nodedb/tests/wire/cases/materialized_sum_cross_shard_expanders.rs b/nodedb/tests/wire/cases/materialized_sum_cross_shard_expanders.rs index 01a657395..0757cee4b 100644 --- a/nodedb/tests/wire/cases/materialized_sum_cross_shard_expanders.rs +++ b/nodedb/tests/wire/cases/materialized_sum_cross_shard_expanders.rs @@ -75,7 +75,7 @@ async fn exec(server: &TestServer, sql: &str) { /// Start a server and create the sum pair and the feed. Accounts `acc` and /// `acc2` start empty. Posting `e0` puts 10 on `acc`, posting `e1` puts 5 on /// `acc`. -async fn sum_fixture() -> TestServer { +pub(super) async fn sum_fixture() -> TestServer { let server = TestServer::start_multicores(CORES).await; exec( &server, @@ -137,7 +137,7 @@ async fn sum_fixture() -> TestServer { } /// Insert one feed row. -async fn feed(server: &TestServer, id: &str, account: &str, amount: &str) { +pub(super) async fn feed(server: &TestServer, id: &str, account: &str, amount: &str) { exec( server, &format!( @@ -148,7 +148,7 @@ async fn feed(server: &TestServer, id: &str, account: &str, amount: &str) { } /// The stored balance of `account`. -async fn balance(server: &TestServer, account: &str) -> String { +pub(super) async fn balance(server: &TestServer, account: &str) -> String { let rows = server .query_text(&format!( "SELECT balance FROM {ACCOUNTS} WHERE id = '{account}'" @@ -161,7 +161,7 @@ async fn balance(server: &TestServer, account: &str) -> String { } } -const INSERT_SELECT: &str = "INSERT INTO ptv_post (id, account_id, amount) \ +pub(super) const INSERT_SELECT: &str = "INSERT INTO ptv_post (id, account_id, amount) \ SELECT id, account_id, amount FROM ptv_feed"; /// The amount of `e0` changes in place. `e1` moves from `acc` to `acc2`. diff --git a/nodedb/tests/wire/cases/materialized_sum_cross_shard_http.rs b/nodedb/tests/wire/cases/materialized_sum_cross_shard_http.rs new file mode 100644 index 000000000..4a48c9746 --- /dev/null +++ b/nodedb/tests/wire/cases/materialized_sum_cross_shard_http.rs @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A plain `INSERT` and an `INSERT ... SELECT` into the source of a +//! materialized sum whose target lives on another vShard, over HTTP +//! `/v1/query`, NDJSON `/v1/query/stream` and WebSocket RPC `/v1/ws`. +//! +//! The plain `INSERT` plans the source write and an `ApplyBalanceDelta` task +//! homed on the target's vShard. The two commit together through Calvin. +//! The `INSERT ... SELECT` runs in an implicit transaction that expands it +//! into point writes and ships each balance to its target. Each route +//! answers one `{"affected": n}` row: the balance task adds no count, as on +//! pgwire and native. +//! +//! The fixture is the one the pgwire cases use: two Data Plane cores, and a +//! target and source that land on different cores. + +use futures::{SinkExt, StreamExt}; +use tokio_tungstenite::tungstenite::Message; + +use super::materialized_sum_cross_shard_expanders::{INSERT_SELECT, balance, feed, sum_fixture}; +use crate::harness::TestServer; + +/// A plain write into the source. It credits 7 to `acc2`. +const PLAIN_INSERT: &str = + "INSERT INTO ptv_post (id, account_id, amount) VALUES ('h1', 'acc2', '7')"; + +/// The answer a write that touched `n` rows gives on these routes. +fn affected(n: u64) -> serde_json::Value { + serde_json::json!({ "affected": n }) +} + +/// Seed the feed `INSERT_SELECT` copies: 4 for `acc`, 6 for `acc2`. +async fn seed_feed(server: &TestServer) { + feed(server, "f1", "acc", "4").await; + feed(server, "f2", "acc2", "6").await; +} + +/// The balances after `PLAIN_INSERT` and `INSERT_SELECT` both ran. +async fn assert_balances(server: &TestServer) { + assert_eq!(balance(server, "acc").await, "19", "10 + 5 + 4"); + assert_eq!(balance(server, "acc2").await, "13", "7 + 6"); +} + +/// POST `sql` to `/v1/query` and return its `rows`. +async fn http_rows(http_port: u16, sql: &str) -> Vec { + let response = reqwest::Client::new() + .post(format!("http://127.0.0.1:{http_port}/v1/query")) + .json(&serde_json::json!({ "sql": sql })) + .send() + .await + .unwrap_or_else(|e| panic!("POST /v1/query {sql}: {e}")); + let status = response.status(); + let body: serde_json::Value = response + .json() + .await + .unwrap_or_else(|e| panic!("decode the /v1/query body for {sql}: {e}")); + assert!(status.is_success(), "{sql} answered {status}: {body}"); + body["rows"] + .as_array() + .unwrap_or_else(|| panic!("a `rows` array for {sql}: {body}")) + .clone() +} + +/// POST `sql` to `/v1/query/stream` and return its lines, parsed. +async fn ndjson_lines(http_port: u16, sql: &str) -> Vec { + let response = reqwest::Client::new() + .post(format!("http://127.0.0.1:{http_port}/v1/query/stream")) + .json(&serde_json::json!({ "sql": sql })) + .send() + .await + .unwrap_or_else(|e| panic!("POST /v1/query/stream {sql}: {e}")); + let status = response.status(); + let body = response + .text() + .await + .unwrap_or_else(|e| panic!("read the /v1/query/stream body for {sql}: {e}")); + assert!(status.is_success(), "{sql} answered {status}: {body}"); + body.lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| { + serde_json::from_str(line) + .unwrap_or_else(|e| panic!("an NDJSON line for {sql}: {e}: {line}")) + }) + .collect() +} + +type WsStream = + tokio_tungstenite::WebSocketStream>; + +/// Send `sql` as RPC `id` and return its `result`. +async fn ws_result(ws: &mut WsStream, id: u64, sql: &str) -> serde_json::Value { + let request = serde_json::json!({ "id": id, "method": "query", "params": { "sql": sql } }); + ws.send(Message::Text(request.to_string().into())) + .await + .unwrap_or_else(|e| panic!("send {sql}: {e}")); + loop { + let message = tokio::time::timeout(std::time::Duration::from_secs(30), ws.next()) + .await + .unwrap_or_else(|_| panic!("no answer to {sql}")) + .unwrap_or_else(|| panic!("the socket closed before answering {sql}")) + .unwrap_or_else(|e| panic!("read the answer to {sql}: {e}")); + let Message::Text(text) = message else { + continue; + }; + let frame: serde_json::Value = serde_json::from_str(&text) + .unwrap_or_else(|e| panic!("a JSON frame for {sql}: {e}: {text}")); + if frame["id"] != id { + continue; + } + assert!(frame.get("error").is_none(), "{sql} failed: {frame}"); + return frame["result"].clone(); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn http_query_moves_a_cross_shard_sum_atomically() { + let server = sum_fixture().await; + seed_feed(&server).await; + + assert_eq!( + http_rows(server.http_port, PLAIN_INSERT).await, + vec![affected(1)], + "one row: the balance task adds no count" + ); + assert_eq!(balance(&server, "acc2").await, "7", "h1 credited"); + + assert_eq!( + http_rows(server.http_port, INSERT_SELECT).await, + vec![affected(2)], + "two copied rows: a balance task adds no count" + ); + assert_balances(&server).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ndjson_query_moves_a_cross_shard_sum_atomically() { + let server = sum_fixture().await; + seed_feed(&server).await; + + assert_eq!( + ndjson_lines(server.http_port, PLAIN_INSERT).await, + vec![affected(1)], + "one line: the balance task adds no count" + ); + assert_eq!(balance(&server, "acc2").await, "7", "h1 credited"); + + assert_eq!( + ndjson_lines(server.http_port, INSERT_SELECT).await, + vec![affected(2)], + "two copied rows: a balance task adds no count" + ); + assert_balances(&server).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ws_rpc_query_moves_a_cross_shard_sum_atomically() { + let server = sum_fixture().await; + seed_feed(&server).await; + let (mut ws, _) = + tokio_tungstenite::connect_async(format!("ws://127.0.0.1:{}/v1/ws", server.http_port)) + .await + .expect("connect the WebSocket RPC endpoint"); + + assert_eq!( + ws_result(&mut ws, 1, PLAIN_INSERT).await, + affected(1), + "one row: the balance task adds no count" + ); + assert_eq!(balance(&server, "acc2").await, "7", "h1 credited"); + + assert_eq!( + ws_result(&mut ws, 2, INSERT_SELECT).await, + affected(2), + "two copied rows: a balance task adds no count" + ); + assert_balances(&server).await; +} diff --git a/nodedb/tests/wire/cases/materialized_sum_txn_preimage.rs b/nodedb/tests/wire/cases/materialized_sum_txn_preimage.rs new file mode 100644 index 000000000..c1295497c --- /dev/null +++ b/nodedb/tests/wire/cases/materialized_sum_txn_preimage.rs @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A write to a materialized-sum source in a transaction block settles its +//! balance from the row as the transaction sees it. +//! +//! An UPDATE or DELETE of a source row owes the target the difference between +//! the row's pre-image and its post-image. When an earlier statement of the +//! same transaction inserted or rewrote that row, the pre-image is the staged +//! row, not the committed one. A pre-image read from committed base alone +//! misses a staged insert and returns the base value of a staged rewrite. The +//! balance is then wrong by the staged row's contribution. +//! +//! Each test runs on both pairs. On the cross-shard pair the balance travels +//! on its own task, settled from the pre-image at statement time. On the +//! co-resident pair the source write folds it at commit, from the target rows +//! the statement resolved off the same pre-image. + +use nodedb_types::id::DatabaseId; + +use crate::harness::TestServer; + +/// A target and source on one vShard: the balance rides the source write. +const CO_RESIDENT: (&str, &str) = ("uo_accounts", "uo_entries"); +/// A target and source on two vShards: the balance travels on its own task. +const CROSS_SHARD: (&str, &str) = ("ptv_acct", "ptv_post"); +/// Data Plane cores the server runs. +const CORES: usize = 2; + +fn vshard(name: &str) -> u32 { + nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, name) + .vshard() + .as_u32() +} + +/// The premise both paths rest on: the names hash as the constants say, and +/// the cross-shard pair lands on two cores. +#[test] +fn the_pairs_hash_to_the_paths_they_name() { + assert_eq!(vshard(CO_RESIDENT.0), vshard(CO_RESIDENT.1)); + assert_ne!( + vshard(CROSS_SHARD.0), + vshard(CROSS_SHARD.1), + "the cross-shard pair must plan a separate balance task" + ); + assert_ne!( + vshard(CROSS_SHARD.0) as usize % CORES, + vshard(CROSS_SHARD.1) as usize % CORES, + "a balance folded on the source's core must find no target row" + ); +} + +async fn exec(server: &TestServer, sql: &str) { + server + .exec(sql) + .await + .unwrap_or_else(|e| panic!("run {sql}: {e}")); +} + +/// Start a server and create both sum pairs. Each pair has accounts `acc-0` +/// and `acc-1`, and posting `e0` puts 10 on `acc-0`. +async fn fixture() -> TestServer { + let server = TestServer::start_multicores(CORES).await; + for (target, source) in [CO_RESIDENT, CROSS_SHARD] { + exec( + &server, + &format!( + "CREATE COLLECTION {target} (id TEXT PRIMARY KEY, owner TEXT) \ + WITH (engine='document_strict')" + ), + ) + .await; + exec( + &server, + &format!( + "CREATE COLLECTION {source} (id TEXT PRIMARY KEY, account_id TEXT, amount TEXT) \ + WITH (engine='document_strict')" + ), + ) + .await; + exec( + &server, + &format!( + "ALTER COLLECTION {target} ADD COLUMN balance TEXT \ + MATERIALIZED_SUM SOURCE {source} \ + ON {source}.account_id = {target}.id VALUE {source}.amount" + ), + ) + .await; + for account in ["acc-0", "acc-1"] { + exec( + &server, + &format!( + "INSERT INTO {target} (id, owner, balance) VALUES ('{account}', 'alice', '0')" + ), + ) + .await; + } + exec( + &server, + &format!("INSERT INTO {source} (id, account_id, amount) VALUES ('e0', 'acc-0', '10')"), + ) + .await; + assert_eq!( + balance(&server, target, "acc-0").await, + "10", + "the seed posting lands on {target}" + ); + } + server +} + +/// The stored balance of `account` in `target`. +async fn balance(server: &TestServer, target: &str, account: &str) -> String { + let rows = server + .query_text(&format!( + "SELECT balance FROM {target} WHERE id = '{account}'" + )) + .await + .unwrap_or_else(|e| panic!("read the balance of {target}.{account}: {e}")); + match rows.as_slice() { + [value] => value.clone(), + other => panic!("one balance row for {target}.{account}, got {other:?}"), + } +} + +/// Run `statements` in one transaction block on every pair, then check each +/// pair's balances of `acc-0` and `acc-1`. +async fn run_block(statements: &[&str], expected: (&str, &str)) { + let server = fixture().await; + for (target, source) in [CO_RESIDENT, CROSS_SHARD] { + let body: Vec = statements + .iter() + .map(|statement| statement.replace("{source}", source)) + .collect(); + let sql = format!("BEGIN; {}; COMMIT", body.join("; ")); + exec(&server, &sql).await; + assert_eq!( + balance(&server, target, "acc-0").await, + expected.0, + "acc-0 of {target} after {sql}" + ); + assert_eq!( + balance(&server, target, "acc-1").await, + expected.1, + "acc-1 of {target} after {sql}" + ); + } +} + +/// A row inserted and deleted in one block leaves the balance unchanged. The +/// delete's pre-image is the staged row, so it debits the amount the insert +/// credited. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_staged_insert_then_delete_leaves_the_balance_unchanged() { + run_block( + &[ + "INSERT INTO {source} (id, account_id, amount) VALUES ('e1', 'acc-0', '5')", + "DELETE FROM {source} WHERE id = 'e1'", + ], + ("10", "0"), + ) + .await; +} + +/// A row inserted and then updated in one block credits the updated amount. +/// The update's pre-image is the staged row, so it moves the difference. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_staged_insert_then_update_credits_the_updated_amount() { + run_block( + &[ + "INSERT INTO {source} (id, account_id, amount) VALUES ('e1', 'acc-0', '5')", + "UPDATE {source} SET amount = '7' WHERE id = 'e1'", + ], + ("17", "0"), + ) + .await; +} + +/// A row moved to another account and then updated in one block: the old +/// account loses the original amount and the new account holds the updated +/// one. The second update's pre-image names the account the first moved the +/// row to. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_staged_move_then_update_settles_on_the_new_account() { + run_block( + &[ + "UPDATE {source} SET account_id = 'acc-1' WHERE id = 'e0'", + "UPDATE {source} SET amount = '9' WHERE id = 'e0'", + ], + ("0", "9"), + ) + .await; +} + +/// A row inserted and then moved to another account in one block credits +/// only the account it ends on. The move's pre-image is the staged row, so +/// the statement resolves and debits the account the insert credited. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_staged_insert_then_move_credits_only_the_final_account() { + run_block( + &[ + "INSERT INTO {source} (id, account_id, amount) VALUES ('e1', 'acc-0', '5')", + "UPDATE {source} SET account_id = 'acc-1' WHERE id = 'e1'", + ], + ("10", "5"), + ) + .await; +} diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index e7b102d2d..96e339a60 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -128,7 +128,9 @@ mod kv_transfer_decimal; mod kv_write_row_level_security; mod kv_write_row_level_security_atomics; mod materialized_sum_cross_shard_expanders; +mod materialized_sum_cross_shard_http; mod materialized_sum_txn_command_tag; +mod materialized_sum_txn_preimage; mod merge_insert_atomicity; mod merge_insert_cross_engine_visibility; mod merge_insert_renamed_source_column; From efcb536c3d4982954baad4e9923cd10d99642f98 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sat, 10 Oct 2026 12:50:03 +0800 Subject: [PATCH 10/19] feat(raft): add a commit fail gate behind the failpoints feature A leader under the gate counts no quorum, so cluster tests can keep an entry uncommitted across a leadership change. The gate is compiled only with the failpoints feature and is forwarded from the nodedb and nodedb-cluster crates. --- Cargo.lock | 1 + nodedb-cluster/Cargo.toml | 2 ++ nodedb-raft/Cargo.toml | 7 +++++++ nodedb-raft/src/fail_gate.rs | 36 ++++++++++++++++++++++++++++++++ nodedb-raft/src/lib.rs | 2 ++ nodedb-raft/src/node/internal.rs | 4 ++++ nodedb/Cargo.toml | 2 +- 7 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 nodedb-raft/src/fail_gate.rs diff --git a/Cargo.lock b/Cargo.lock index 55f8abdb3..1fa939bb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4594,6 +4594,7 @@ name = "nodedb-raft" version = "0.5.0" dependencies = [ "crc32c", + "nodedb-types", "rand 0.10.2", "rkyv", "serde", diff --git a/nodedb-cluster/Cargo.toml b/nodedb-cluster/Cargo.toml index a3bf676cd..eb0b0403a 100644 --- a/nodedb-cluster/Cargo.toml +++ b/nodedb-cluster/Cargo.toml @@ -19,6 +19,8 @@ default = [] # `faultbox::init`; a no-op on wasm32 regardless. Off by default so the crate # stays dependency-light for embedders that report failures their own way. diagnostics = ["dep:faultbox"] +# Arms the Raft fail gates cluster tests use. Off by default. +failpoints = ["nodedb-raft/failpoints"] [dependencies] # Workspace crates diff --git a/nodedb-raft/Cargo.toml b/nodedb-raft/Cargo.toml index 8f43ac9e2..623b44fc9 100644 --- a/nodedb-raft/Cargo.toml +++ b/nodedb-raft/Cargo.toml @@ -12,8 +12,15 @@ documentation = "https://docs.rs/nodedb-raft" keywords = ["raft", "consensus", "replication", "distributed", "leader-election"] categories = ["algorithms", "network-programming"] +[features] +default = [] +# Arms the commit gate (`raft::hold_commit::node::group`) cluster tests +# use to keep an entry uncommitted across a leadership change. +failpoints = ["dep:nodedb-types", "nodedb-types/failpoints"] + [dependencies] # Workspace +nodedb-types = { workspace = true, optional = true } serde = { workspace = true } serde_json = { workspace = true } sonic-rs = { workspace = true } diff --git a/nodedb-raft/src/fail_gate.rs b/nodedb-raft/src/fail_gate.rs new file mode 100644 index 000000000..69809d65d --- /dev/null +++ b/nodedb-raft/src/fail_gate.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Fail gates for Raft tests. +//! +//! Compiled only with the `failpoints` feature. A gate answers without +//! waiting: Raft steps are synchronous, so a gate holds a step back and the +//! driver asks again on its next step. + +use nodedb_types::fail_point::{FailAction, lookup}; + +/// Whether the commit gate of `group_id` on `node_id` holds now. Named +/// `raft::hold_commit::node::group`, armed with `wait_file()`. +/// It holds while the file is absent. +/// +/// A leader under the gate counts no quorum, so it commits no new entry. +/// The entries it appends still replicate. A test keeps an entry +/// uncommitted this way, then moves the leadership: the entry commits only +/// in the next term. On its first hold the gate creates `.parked`. +pub(crate) fn holds_commit(node_id: u64, group_id: u64) -> bool { + let name = format!("raft::hold_commit::node{node_id}::group{group_id}"); + let Some(FailAction::WaitForFile(path)) = lookup(&name) else { + return false; + }; + if path.exists() { + return false; + } + let mut parked = path.into_os_string(); + parked.push(".parked"); + let parked = std::path::PathBuf::from(parked); + if !parked.exists() + && let Err(e) = std::fs::write(&parked, &name) + { + tracing::warn!(gate = %name, error = %e, "fail gate parked marker not created"); + } + true +} diff --git a/nodedb-raft/src/lib.rs b/nodedb-raft/src/lib.rs index a7e7f0e2a..203c93d80 100644 --- a/nodedb-raft/src/lib.rs +++ b/nodedb-raft/src/lib.rs @@ -10,6 +10,8 @@ //! catalog and the cross-engine surrogate counter). pub mod error; +#[cfg(feature = "failpoints")] +mod fail_gate; pub mod log; pub mod message; pub mod node; diff --git a/nodedb-raft/src/node/internal.rs b/nodedb-raft/src/node/internal.rs index dc1cf1fa1..cc9eb796a 100644 --- a/nodedb-raft/src/node/internal.rs +++ b/nodedb-raft/src/node/internal.rs @@ -407,6 +407,10 @@ impl RaftNode { /// up for promotion) but intentionally excluded from this calculation /// so adding a learner never weakens the commit quorum. pub(super) fn try_advance_commit_index(&mut self) { + #[cfg(feature = "failpoints")] + if crate::fail_gate::holds_commit(self.config.node_id, self.config.group_id) { + return; + } let leader = match &self.leader_state { Some(ls) => ls, None => return, diff --git a/nodedb/Cargo.toml b/nodedb/Cargo.toml index c9cf82312..a45320c78 100644 --- a/nodedb/Cargo.toml +++ b/nodedb/Cargo.toml @@ -188,4 +188,4 @@ io-uring = { workspace = true } default = [] # Enables crash-injection points used by recovery tests. Off by default; tests # that exercise the framework opt in via `--features failpoints`. -failpoints = ["nodedb-types/failpoints", "nodedb-wal/failpoints"] +failpoints = ["nodedb-types/failpoints", "nodedb-wal/failpoints", "nodedb-cluster/failpoints"] From a7647be7d188cc51dd427e0095b92175ebcc5bf1 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sat, 10 Oct 2026 12:50:03 +0800 Subject: [PATCH 11/19] feat(calvin): restage committed transactions after a failed leader stage A leader whose own stage fails after an earlier leader's COMMIT vote decided the transaction now stages it again with doubling backoff instead of dropping it. The scheduler halts once the attempts run out. Adds restage_attempts and restage_backoff_ms tuning knobs and restage metrics. --- docs/getting-started.md | 7 + nodedb-types/src/config/tuning/calvin.rs | 26 + nodedb/src/config/server/config.rs | 13 + nodedb/src/config/server/domain.rs | 8 + .../calvin/scheduler/driver/barrier.rs | 7 +- .../cluster/calvin/scheduler/driver/config.rs | 43 ++ .../driver/core/commit_resolve/verdict.rs | 62 +- .../driver/core/commit_resolve/vote.rs | 2 +- .../scheduler/driver/core/completion_route.rs | 3 +- .../calvin/scheduler/driver/core/deferred.rs | 12 +- .../driver/core/dispatch/incarnation.rs | 10 +- .../driver/core/dispatch/unstaged.rs | 6 +- .../calvin/scheduler/driver/core/halt.rs | 7 +- .../calvin/scheduler/driver/core/mod.rs | 7 +- .../calvin/scheduler/driver/core/process.rs | 4 +- .../scheduler/driver/core/read_result.rs | 159 ++++- .../scheduler/driver/core/redo_applied.rs | 9 +- .../scheduler/driver/core/redo_propose.rs | 3 +- .../calvin/scheduler/driver/core/restage.rs | 548 ++++++++++++++++++ .../calvin/scheduler/driver/core/role.rs | 15 +- .../calvin/scheduler/driver/core/run_loop.rs | 10 + .../calvin/scheduler/driver/core/scheduler.rs | 11 + .../cluster/calvin/scheduler/driver/types.rs | 9 +- .../cluster/calvin/scheduler/metrics.rs | 8 + .../cluster/calvin/scheduler/metrics_flow.rs | 22 +- .../cluster/start_raft/observability.rs | 7 + nodedb/src/control/state/calvin_counters.rs | 3 + nodedb/src/control/state/calvin_local.rs | 1 + nodedb/src/control/state/fields.rs | 2 + nodedb/src/control/state/init.rs | 1 + nodedb/src/control/state/init_prod/open.rs | 1 + 31 files changed, 963 insertions(+), 63 deletions(-) create mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/restage.rs diff --git a/docs/getting-started.md b/docs/getting-started.md index 88306b15a..b33cd2215 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -599,6 +599,8 @@ Below that the sweep costs more than the resolution it buys. | `tuning.calvin.verdict_stall_warn_ms` | `5000` | | `tuning.calvin.max_inflight_backlog` | `1024` | | `tuning.calvin.catch_up_window` | `512` | +| `tuning.calvin.restage_attempts` | `5` | +| `tuning.calvin.restage_backoff_ms` | `100` | Each hosted vShard runs one Calvin scheduler with these settings. @@ -612,6 +614,11 @@ Each hosted vShard runs one Calvin scheduler with these settings. taking new input. - `catch_up_window` is the most sequencer log entries one catch-up pass replays. +- `restage_attempts` is the most times a data-group leader stages a + committed transaction again after its own stage failed. The scheduler + halts once they run out. +- `restage_backoff_ms` is the wait before the first restage. Each later + restage waits twice as long, at most 60 seconds. Every value must be positive. `verdict_stall_warn_ms` has a floor of `4`. diff --git a/nodedb-types/src/config/tuning/calvin.rs b/nodedb-types/src/config/tuning/calvin.rs index 107570077..13ac0e585 100644 --- a/nodedb-types/src/config/tuning/calvin.rs +++ b/nodedb-types/src/config/tuning/calvin.rs @@ -48,6 +48,15 @@ fn default_max_open_redo_bytes() -> u64 { 512 * 1024 * 1024 } +fn default_restage_attempts() -> u32 { + 5 +} + +fn default_restage_backoff_ms() -> u64 { + // Five sequencer epochs of 20 ms. Five doubling waits span about 3 s. + 100 +} + /// Tuning knobs for the Calvin scheduler that runs per hosted vShard. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CalvinTuning { @@ -91,6 +100,19 @@ pub struct CalvinTuning { /// holds at most this much per data group it hosts. #[serde(default = "default_max_open_redo_bytes")] pub max_open_redo_bytes: u64, + + /// Most times a leader stages a committed transaction again after its + /// own stage failed. An earlier leader's COMMIT vote decided the + /// transaction, so it cannot drop. The scheduler halts once the stages + /// run out. + #[serde(default = "default_restage_attempts")] + pub restage_attempts: u32, + + /// Wait in milliseconds before the first restage of a committed + /// transaction. Each later restage waits twice as long as the one + /// before, at most 60 seconds. + #[serde(default = "default_restage_backoff_ms")] + pub restage_backoff_ms: u64, } impl Default for CalvinTuning { @@ -104,6 +126,8 @@ impl Default for CalvinTuning { catch_up_window: default_catch_up_window(), max_redo_entry_bytes: default_max_redo_entry_bytes(), max_open_redo_bytes: default_max_open_redo_bytes(), + restage_attempts: default_restage_attempts(), + restage_backoff_ms: default_restage_backoff_ms(), } } } @@ -123,6 +147,8 @@ mod tests { assert_eq!(t.catch_up_window, 512); assert_eq!(t.max_redo_entry_bytes, 8 * 1024 * 1024); assert_eq!(t.max_open_redo_bytes, 512 * 1024 * 1024); + assert_eq!(t.restage_attempts, 5); + assert_eq!(t.restage_backoff_ms, 100); } #[test] diff --git a/nodedb/src/config/server/config.rs b/nodedb/src/config/server/config.rs index 265e1b607..b83609e69 100644 --- a/nodedb/src/config/server/config.rs +++ b/nodedb/src/config/server/config.rs @@ -447,6 +447,19 @@ mod tests { assert!(msg.contains("positive integer"), "{msg}"); } + #[test] + fn from_file_rejects_a_zero_calvin_restage_bound() { + let path = write_temp_config( + "nodedb-domain-calvin-restage.toml", + "[tuning.calvin]\nrestage_attempts = 0\n", + ); + let err = ServerConfig::from_file(&path).unwrap_err(); + std::fs::remove_file(&path).ok(); + let msg = err.to_string(); + assert!(msg.contains("tuning.calvin.restage_attempts"), "{msg}"); + assert!(msg.contains("positive integer"), "{msg}"); + } + #[test] fn from_file_rejects_a_calvin_stall_warning_below_the_floor() { let path = write_temp_config( diff --git a/nodedb/src/config/server/domain.rs b/nodedb/src/config/server/domain.rs index cba06ab96..b152a1300 100644 --- a/nodedb/src/config/server/domain.rs +++ b/nodedb/src/config/server/domain.rs @@ -173,6 +173,14 @@ fn validate_calvin(config: &ServerConfig) -> crate::Result<()> { "tuning.calvin.max_inflight_backlog", )?; positive_u64(calvin.catch_up_window, "tuning.calvin.catch_up_window")?; + positive_u64( + u64::from(calvin.restage_attempts), + "tuning.calvin.restage_attempts", + )?; + positive_u64( + calvin.restage_backoff_ms, + "tuning.calvin.restage_backoff_ms", + )?; validate_redo_sizes(calvin) } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/barrier.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/barrier.rs index eee382569..ec202df1c 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/barrier.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/barrier.rs @@ -50,6 +50,10 @@ pub struct ReadResultEvent { pub values: Vec<(PassiveReadKeyId, Value)>, } +/// Read values received from passive vShards, keyed by passive vShard. +/// `BTreeMap` for determinism. +pub type ReceivedReads = BTreeMap>; + // ── PendingDependentBarrier ──────────────────────────────────────────────────── /// A dependent-read barrier: tracks which passive vshards have delivered @@ -66,8 +70,7 @@ pub struct PendingDependentBarrier { /// `BTreeSet` for determinism. pub waiting_for: BTreeSet, /// Received read values, keyed by passive vshard. - /// `BTreeMap` for determinism. - pub received: BTreeMap>, + pub received: ReceivedReads, /// Deadline for passive participant delivery. /// /// `Instant::now()` used for barrier timeout (observability / off-WAL path). diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/config.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/config.rs index 32671d071..85ede01c4 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/config.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/config.rs @@ -10,6 +10,9 @@ use std::time::Duration; use nodedb_cluster::calvin::SequencerConfig; use nodedb_types::config::tuning::CalvinTuning; +/// The longest wait between two restages of one committed txn. +pub const MAX_RESTAGE_BACKOFF: Duration = Duration::from_secs(60); + /// Tuning parameters for a [`super::core::Scheduler`] instance. #[derive(Debug, Clone)] pub struct SchedulerConfig { @@ -54,6 +57,17 @@ pub struct SchedulerConfig { /// the fan-out channel is full, so one window covers about one channel of /// missed entries. Windows run back to back while intake is open. pub catch_up_window: u64, + /// Most restages of a committed txn whose stage failed on this leader. + /// The scheduler halts once they run out. + /// + /// Default: 5. + pub restage_attempts: u32, + /// Wait before the first restage of a committed txn. Each later restage + /// waits twice as long. The wait only delays a retry of a decided + /// verdict, so it never changes what any replica applies. + /// + /// Default: 100 milliseconds, five default epochs. + pub restage_backoff_ms: u64, } impl Default for SchedulerConfig { @@ -75,6 +89,8 @@ impl SchedulerConfig { verdict_stall_warn_ms: tuning.verdict_stall_warn_ms, max_inflight_backlog: tuning.max_inflight_backlog, catch_up_window: tuning.catch_up_window, + restage_attempts: tuning.restage_attempts, + restage_backoff_ms: tuning.restage_backoff_ms, } } @@ -87,6 +103,15 @@ impl SchedulerConfig { pub fn verdict_stall_warn(&self) -> Duration { Duration::from_millis(self.verdict_stall_warn_ms) } + + /// The wait before restage number `done + 1`, after `done` restages: + /// the base wait doubled once per restage done, at most + /// [`MAX_RESTAGE_BACKOFF`]. + pub fn restage_backoff(&self, done: u32) -> Duration { + let factor = 1u64.checked_shl(done).unwrap_or(u64::MAX); + Duration::from_millis(self.restage_backoff_ms.saturating_mul(factor)) + .min(MAX_RESTAGE_BACKOFF) + } } #[cfg(test)] @@ -103,6 +128,20 @@ mod tests { assert_eq!(config.dependent_read_passive_timeout_ms, 60); assert_eq!(config.verdict_stall_warn_ms, 5_000); assert_eq!(config.catch_up_window, 512); + assert_eq!(config.restage_attempts, 5); + assert_eq!(config.restage_backoff_ms, 100); + } + + /// Each restage waits twice as long as the one before, up to the cap. + /// A count past the shift width saturates instead of wrapping. + #[test] + fn the_restage_backoff_doubles_per_restage() { + let config = SchedulerConfig::default(); + assert_eq!(config.restage_backoff(0), Duration::from_millis(100)); + assert_eq!(config.restage_backoff(1), Duration::from_millis(200)); + assert_eq!(config.restage_backoff(4), Duration::from_millis(1_600)); + assert_eq!(config.restage_backoff(20), MAX_RESTAGE_BACKOFF); + assert_eq!(config.restage_backoff(64), MAX_RESTAGE_BACKOFF); } /// The tuning default restates the dispatcher queue capacity, which the @@ -124,6 +163,8 @@ mod tests { let tuning = CalvinTuning { max_inflight_backlog: 128, verdict_stall_warn_ms: 9_000, + restage_attempts: 2, + restage_backoff_ms: 7, ..CalvinTuning::default() }; let sequencer = SequencerConfig { @@ -134,5 +175,7 @@ mod tests { assert_eq!(config.max_inflight_backlog, 128); assert_eq!(config.verdict_stall_warn(), Duration::from_millis(9_000)); assert_eq!(config.epoch_duration_ms, 50); + assert_eq!(config.restage_attempts, 2); + assert_eq!(config.restage_backoff(0), Duration::from_millis(7)); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs index 334f34e86..ebe401cac 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs @@ -12,7 +12,6 @@ use nodedb_cluster::calvin::VerdictSignal; use crate::control::cluster::calvin::scheduler::driver::core::deferred::{ DispatchOutcome, DispatchStep, }; -use crate::control::cluster::calvin::scheduler::driver::core::halt::{HaltReason, HaltStep}; use crate::control::cluster::calvin::scheduler::driver::core::owed::SchedulerProposal; use crate::control::cluster::calvin::scheduler::driver::core::process::LedgerMark; use crate::control::cluster::calvin::scheduler::driver::core::scheduler::Scheduler; @@ -40,9 +39,9 @@ impl Scheduler { /// Any other state already left the barrier, so a duplicate push, probe /// or sweep is a no-op. /// - /// A COMMIT verdict for a txn this leader failed to stage halts the - /// scheduler: the txn stays parked with its locks, its stall deadline - /// cleared, and its position unapplied. + /// A COMMIT verdict for a txn this leader failed to stage restages it + /// (see `super::super::restage`). The txn keeps its locks and its + /// position stays unapplied meanwhile. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn resume_on_verdict( &mut self, txn_id: TxnId, @@ -66,23 +65,22 @@ impl Scheduler { | CommitState::AwaitingResolveTurn | CommitState::AwaitingRedoResolve | CommitState::AwaitingRedoApply { .. } - | CommitState::AwaitingDrop => {} + | CommitState::AwaitingDrop + | CommitState::AwaitingRestage => {} } } /// Resume a leader's txn parked on the barrier. fn resume_parked(&mut self, txn_id: TxnId, committed: bool, writes: bool) { + // An earlier leader of the vShard voted COMMIT, and this leader's + // stage failed: the txn cannot drop while its peers apply it. if committed - && let Some(pending) = self.pending.get_mut(&txn_id) - && let Some(stage_error) = pending.stage_error.clone() + && self + .pending + .get(&txn_id) + .is_some_and(|pending| pending.stage_error.is_some()) { - pending.verdict_deadline = None; - self.halt_apply( - txn_id, - HaltReason::LocalStageFailed, - HaltStep::Stage, - format!("COMMIT verdict for a txn this leader did not stage: {stage_error}"), - ); + self.restage_or_halt(txn_id); return; } if let Some(pending) = self.pending.get_mut(&txn_id) { @@ -247,7 +245,6 @@ mod tests { use crate::bridge::dispatch::CoreChannelDataSide; use crate::bridge::envelope::ErrorCode; use crate::bridge::envelope::{Payload, StageVote, Status}; - use crate::control::cluster::calvin::scheduler::driver::core::halt::HaltReason; use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::lead_data_group; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ await_data_plane_request, build_test_scheduler, build_test_scheduler_with_data_side, @@ -561,12 +558,12 @@ mod tests { (scheduler, dir, data_side) } - /// A COMMIT verdict for a txn this leader failed to stage halts the - /// scheduler: an earlier leader staged and voted commit, so this leader - /// holds nothing to resolve. No resolve is dispatched, and the stall - /// sweep stops. + /// A COMMIT verdict for a txn this leader failed to stage holds it for a + /// restage: an earlier leader staged and voted commit, so the txn cannot + /// drop. The failed stage's state is discarded, no resolve is + /// dispatched, and the position stays unapplied. #[tokio::test] - async fn commit_verdict_after_local_stage_error_halts_unapplied() { + async fn commit_verdict_after_local_stage_error_awaits_a_restage() { let txn_id = TxnId::new(14, 2); let (mut scheduler, _dir, mut data_side) = leader_with_failed_stage(txn_id); @@ -577,12 +574,26 @@ mod tests { .pending .get(&txn_id) .expect("the txn stays pending"); - assert_eq!(pending.commit_state, CommitState::AwaitingVerdict); + assert_eq!(pending.commit_state, CommitState::AwaitingRestage); assert_eq!(pending.verdict_deadline, None); - assert_eq!( - scheduler.apply_halt().map(|h| h.reason), - Some(HaltReason::LocalStageFailed) - ); + assert!(!scheduler.is_apply_halted()); + let restage = scheduler + .restages + .get(&txn_id) + .expect("the txn waits for its first restage"); + assert_eq!(restage.attempts, 0); + assert!(restage.due.is_some()); + let request = data_side + .request_rx + .try_pop() + .expect("the failed stage's state is discarded"); + assert!(matches!( + request.inner.plan, + PhysicalPlan::Meta(MetaOp::CalvinDrop { + epoch: 14, + position: 2 + }) + )); assert!( data_side.request_rx.try_pop().is_err(), "no resolve reaches the Data Plane" @@ -599,6 +610,7 @@ mod tests { scheduler.resume_on_verdict(txn_id, false); assert!(!scheduler.is_apply_halted()); + assert!(scheduler.restages.is_empty(), "an abort never restages"); let request = data_side .request_rx .try_pop() diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs index 97d6acb33..978a94126 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/vote.rs @@ -112,7 +112,7 @@ impl Scheduler { // A stage error parks too. The leader votes abort, and every // participant drops at the abort verdict. A COMMIT verdict for it // can follow only a vote an earlier leader cast: `resume_on_verdict` - // halts on it. + // restages the txn. match self.pending.get_mut(&txn_id) { Some(pending) => { pending.commit_state = CommitState::AwaitingVerdict; diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs index 0f0049ec7..dd1a0d0c2 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs @@ -86,7 +86,8 @@ impl Scheduler { CommitState::Following | CommitState::AwaitingVerdict | CommitState::AwaitingResolveTurn - | CommitState::AwaitingRedoApply { .. } => { + | CommitState::AwaitingRedoApply { .. } + | CommitState::AwaitingRestage => { // A txn in these states sends no request it waits for. A // recorded request names a step it left. tracing::warn!( diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs index 99779c575..959df7b06 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs @@ -32,8 +32,8 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) enum DispatchSt /// completes on its answer. Drop, /// `CalvinDrop` of staged state the txn no longer needs: a demoted - /// leader's, or a restaged slice whose redo applied. Its answer is - /// drained and read by nothing. + /// leader's, a failed stage a restage replaces, or a restaged slice + /// whose redo applied. Its answer is drained and read by nothing. Discard, } @@ -119,6 +119,14 @@ impl Scheduler { !self.deferred.is_empty() } + /// Whether a refused request of `txn_id` waits for capacity. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn has_deferred_for( + &self, + txn_id: TxnId, + ) -> bool { + self.deferred.iter().any(|parked| parked.txn_id == txn_id) + } + /// Whether the run loop re-sends parked requests: some wait, and the /// scheduler has not halted. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn resends_deferred( diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs index 42968d41d..0af735c75 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/incarnation.rs @@ -243,9 +243,10 @@ mod tests { drop(reclaim); } - /// A leader that halts on a txn it never staged gives back the gates the - /// txn held: a purge of its collection proceeds. The txn checks again - /// before a proposal and fails once the collection was recreated. + /// A leader that halts on a committed txn whose collection it found + /// superseded gives back the gates the txn held: a purge of its + /// collection proceeds. The txn checks again before a proposal and fails + /// once the collection was recreated. #[tokio::test] async fn a_halt_releases_the_gates_and_a_later_proposal_checks_again() { let txn_id = TxnId::new(5, 0); @@ -270,7 +271,8 @@ mod tests { if let Some(pending) = scheduler.pending.get_mut(&txn_id) { pending.txn.tx_class = tx_class; pending.gates = gates; - pending.stage_error = Some("stage refused on this replica".into()); + pending.superseded = true; + pending.stage_error = Some("a collection was superseded on this replica".into()); } scheduler.resume_on_verdict(txn_id, true); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs index 57ea07a1f..c02d6e001 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/unstaged.rs @@ -60,9 +60,9 @@ impl Scheduler { /// Park `txn`, which never staged on this replica, on the commit barrier /// with an abort vote for `reason`. /// - /// `stage_error` stays on the pending entry. A COMMIT verdict for the - /// txn then halts the scheduler, because this leader holds nothing to - /// resolve. + /// `stage_error` stays on the pending entry. This leader holds nothing + /// to resolve, so a COMMIT verdict restages the txn through the grant + /// path: the plans route again, or the barrier opens again. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn park_unstaged( &mut self, txn: SequencedTxn, diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs index 13ee42747..435dfd721 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs @@ -51,7 +51,8 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) enum HaltReason ResponseDisconnected, /// The resolve of a committed txn failed or returned an undecodable record. ResolveFailed, - /// This leader failed to stage a txn whose verdict is COMMIT. + /// This leader cannot stage a txn whose verdict is COMMIT: its + /// restages ran out, or a collection it names was superseded. LocalStageFailed, /// The surrogate catalog refused a coordinator-assigned identity. IdentityBindFailed, @@ -140,7 +141,9 @@ impl HaltStep { state: CommitState, ) -> Self { match state { - CommitState::Staged | CommitState::AwaitingVerdict => Self::Stage, + CommitState::Staged | CommitState::AwaitingVerdict | CommitState::AwaitingRestage => { + Self::Stage + } CommitState::AwaitingRedoResolve | CommitState::AwaitingResolveTurn => Self::Resolve, CommitState::AwaitingRedoApply { .. } | CommitState::Following => Self::RedoApply, CommitState::AwaitingDrop => Self::Drop, diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs index 47a2c510f..a77330fe6 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs @@ -13,9 +13,9 @@ //! //! All bookkeeping uses `BTreeMap`/`BTreeSet` — never `HashMap`/`HashSet` — //! and dispatch order is `(epoch, position)` order. `Instant::now()` is used -//! only for lock-wait latency metrics and the dependent-read barrier -//! `timeout_at`, both off the WAL-influencing path; every call site carries a -//! `// no-determinism:` marker. +//! only for lock-wait latency metrics, the dependent-read barrier +//! `timeout_at`, and the restage backoff, all off the WAL-influencing path; +//! every call site carries a `// no-determinism:` marker. mod apply_result; pub mod catch_up; @@ -41,6 +41,7 @@ pub mod redo_applied; pub mod redo_propose; pub mod request; pub mod resolve_turn; +mod restage; pub mod role; pub mod routing; pub mod run_loop; diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs index d8455ebc5..a8d24a437 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs @@ -296,7 +296,7 @@ impl Scheduler { timeout_at, }; - self.dependent_barrier.insert(txn_id, barrier); + self.open_barrier(txn_id, barrier); } /// Complete an in-flight txn. @@ -342,6 +342,8 @@ impl Scheduler { lock_owner: TxnId, mark: LedgerMark, ) { + // The txn restages no more, and no barrier of it opens again. + self.forget_held_state(txn_id); // Release this txn's locks. `release` promotes any waiter queued behind // each freed key to holder (moving it pending -> held) and returns the // fully-promoted ids. Those ids are already holders in the table the diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs index 92bfbbd46..054126a7f 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs @@ -1,10 +1,15 @@ // SPDX-License-Identifier: BUSL-1.1 //! `CalvinReadResult` handling and dependent-read barrier timeout sweeps. +//! +//! A read result arrives once, from the data-group log. A dependent txn this +//! scheduler holds with no barrier keeps the results it received: a +//! follower's held txn, or a leader's txn whose barrier timed out. A later +//! barrier of the txn, after a promotion or a restage, starts from them. use tracing::warn; -use super::super::barrier::ReadResultEvent; +use super::super::barrier::{PendingDependentBarrier, ReadResultEvent, ReceivedReads}; use super::scheduler::Scheduler; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; @@ -19,8 +24,15 @@ impl Scheduler { let barrier = match self.dependent_barrier.get_mut(&txn_id) { Some(b) => b, None => { - // No barrier for this txn — can have already timed out or - // been dispatched. Log and ignore. + if self.holds_dependent(txn_id) { + self.hold_reads( + txn_id, + ReceivedReads::from([(event.passive_vshard, event.values)]), + ); + return; + } + // Neither a barrier nor a held txn: the txn finished, or its + // input has not been granted here. Log and ignore. warn!( vshard_id = self.vshard_id, epoch = event.epoch, @@ -43,12 +55,53 @@ impl Scheduler { let Some(barrier) = self.dependent_barrier.remove(&txn_id) else { return; }; + self.dispatch_complete_barrier(txn_id, barrier); + } + /// Open `barrier` for `txn_id`, starting from the reads the txn holds. + /// A barrier those reads complete dispatches at once. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn open_barrier( + &mut self, + txn_id: TxnId, + mut barrier: PendingDependentBarrier, + ) { + if let Some(held) = self.held_reads.remove(&txn_id) { + for (passive_vshard, values) in held { + barrier.waiting_for.remove(&passive_vshard); + barrier.received.insert(passive_vshard, values); + } + } + if barrier.is_complete() { + self.dispatch_complete_barrier(txn_id, barrier); + } else { + self.dependent_barrier.insert(txn_id, barrier); + } + } + + /// Dispatch the active stage of a barrier every passive vShard answered. + fn dispatch_complete_barrier(&mut self, txn_id: TxnId, barrier: PendingDependentBarrier) { let injected_reads = barrier.assemble_injected_reads(); - let lock_owner = barrier.lock_owner; - let txn = barrier.txn; + self.dispatch_active_txn(barrier.txn, txn_id, barrier.lock_owner, injected_reads); + } + + /// Keep `reads` for `txn_id`, a dependent txn in `pending`. Reads for a + /// txn that left `pending` are dropped: it finished. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn hold_reads( + &mut self, + txn_id: TxnId, + reads: ReceivedReads, + ) { + if reads.is_empty() || !self.holds_dependent(txn_id) { + return; + } + self.held_reads.entry(txn_id).or_default().extend(reads); + } - self.dispatch_active_txn(txn, txn_id, lock_owner, injected_reads); + /// Whether `txn_id` is a dependent txn in `pending`. + fn holds_dependent(&self, txn_id: TxnId) -> bool { + self.pending + .get(&txn_id) + .is_some_and(|pending| pending.txn.tx_class.dependent_reads.is_some()) } /// Check all pending dependent barriers for timeout. @@ -91,6 +144,9 @@ impl Scheduler { nodedb_cluster::calvin::AbortReason::ParticipantError, stage_error, ); + // A restage under a COMMIT verdict opens a barrier again. It + // starts from what this one received. + self.hold_reads(txn_id, barrier.received); } } } @@ -111,7 +167,8 @@ mod tests { CapturingProposer, lead_data_group, }; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ - build_test_scheduler_with_data_side, make_sequenced_txn, + build_test_scheduler_with_data_side, make_local_write_txn, make_sequenced_txn, + test_coll_vshard, }; use crate::control::cluster::calvin::scheduler::driver::types::CommitState; @@ -183,4 +240,92 @@ mod tests { ); assert!(!scheduler.applied.is_applied(40, 0)); } + + /// A barrier that timed out keeps what it received, and a read result + /// that lands after the timeout is kept too. Under a COMMIT verdict the + /// restage opens a barrier from those reads. They complete it, so the + /// active stage dispatches at once. + #[tokio::test] + async fn a_restaged_barrier_starts_from_the_reads_it_received() { + use nodedb_cluster::calvin::types::DependentReadSpec; + use nodedb_physical::physical_plan::PhysicalPlan; + use nodedb_physical::physical_plan::meta::MetaOp; + + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side( + test_coll_vshard(), + std::sync::Arc::clone(®istry), + ); + lead_data_group(&mut scheduler); + let txn_id = TxnId::new(41, 0); + let mut txn = make_local_write_txn(41, 0); + txn.tx_class.dependent_reads = Some(DependentReadSpec { + passive_reads: BTreeMap::from([(9, Vec::new()), (10, Vec::new())]), + }); + scheduler.dependent_barrier.insert( + txn_id, + PendingDependentBarrier { + txn, + lock_owner: txn_id, + waiting_for: BTreeSet::from([10]), + received: BTreeMap::from([(9, Vec::new())]), + // no-determinism: test-only deadline already passed. + timeout_at: Instant::now() - Duration::from_millis(1), + }, + ); + + scheduler.check_dependent_barrier_timeouts(); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingVerdict) + ); + scheduler.handle_read_result(ReadResultEvent { + epoch: 41, + position: 0, + passive_vshard: 10, + tenant_id: crate::types::TenantId::new(1), + values: Vec::new(), + }); + assert_eq!( + scheduler + .held_reads + .get(&txn_id) + .map(|held| held.keys().copied().collect::>()), + Some(vec![9, 10]) + ); + + registry.note_verdict( + nodedb_cluster::calvin::TxnId::new(41, 0), + VerdictOutcome::Commit, + ); + scheduler.resume_on_verdict(txn_id, true); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingRestage) + ); + if let Some(restage) = scheduler.restages.get_mut(&txn_id) { + // no-determinism: test-only backoff end in the past. + restage.due = Some(Instant::now() - Duration::from_millis(1)); + } + scheduler.restage_due(); + + let mut staged_active = false; + while let Ok(request) = data_side.request_rx.try_pop() { + staged_active |= matches!( + request.inner.plan, + PhysicalPlan::Meta(MetaOp::CalvinExecuteActive { + epoch: 41, + position: 0, + .. + }) + ); + } + assert!(staged_active, "the restaged barrier completes at once"); + assert!(scheduler.dependent_barrier.is_empty()); + assert!(scheduler.held_reads.is_empty()); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Staged) + ); + } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_applied.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_applied.rs index 1728f46bf..904be3dd5 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_applied.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_applied.rs @@ -157,9 +157,12 @@ fn holds_staged_state(state: CommitState) -> bool { | CommitState::AwaitingResolveTurn | CommitState::AwaitingRedoResolve | CommitState::AwaitingDrop => true, - // The install consumed the staged entry of a proposed slice, and a - // follower staged nothing. - CommitState::AwaitingRedoApply { .. } | CommitState::Following => false, + // The install consumed the staged entry of a proposed slice, a + // follower staged nothing, and a txn awaiting its restage discarded + // what it staged. + CommitState::AwaitingRedoApply { .. } + | CommitState::Following + | CommitState::AwaitingRestage => false, } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs index 64f8e8434..a40add2c9 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs @@ -244,7 +244,8 @@ impl Scheduler { | CommitState::AwaitingVerdict | CommitState::AwaitingResolveTurn | CommitState::AwaitingRedoResolve - | CommitState::AwaitingDrop => None, + | CommitState::AwaitingDrop + | CommitState::AwaitingRestage => None, }) .collect(); for (txn_id, proposed) in owed { diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/restage.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/restage.rs new file mode 100644 index 000000000..3d25c9e54 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/restage.rs @@ -0,0 +1,548 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Restage of a committed txn whose stage failed on this leader. +//! +//! A COMMIT verdict reaches a leader whose own stage failed only when an +//! earlier leader of the vShard voted COMMIT first. The txn cannot drop: +//! its peers apply it. So the leader discards what the failed stage left, +//! holds the txn with its locks, and stages it again once a backoff passes. +//! +//! - A restage takes the path a promotion takes, +//! [`Scheduler::route_granted`]: a static stage, a dependent barrier, or a +//! plan rejection. A rejected plan or a timed-out barrier parks the txn +//! unstaged again. That counts as one more failed stage. +//! - The verdict is known, so the restage's vote changes nothing, and its +//! park probe resumes the txn at once. A restage that stages resolves and +//! proposes its redo like any committed slice. +//! - Due restages run in sequencer order. A txn waits for every lower txn +//! that also waits for its restage. +//! - The leader halts with `LocalStageFailed` once the restages run out. A +//! txn whose collection is superseded cannot commit here, and halts at +//! once. +//! +//! The backoff runs on this node's clock. It only delays a retry of a +//! decided verdict, so no replica applies anything different. + +use std::sync::atomic::Ordering; +use std::time::Instant; + +use tracing::{info, warn}; + +use super::super::types::{CommitState, PendingTxn}; +use super::halt::{HaltReason, HaltStep}; +use super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + +/// How far a committed txn is through its restages. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::control::cluster::calvin::scheduler::driver::core) struct Restage { + /// Restages dispatched so far. + pub attempts: u32, + /// When the next restage dispatches. `None` while one is in flight. + pub due: Option, +} + +impl Scheduler { + /// Act on a COMMIT verdict for `txn_id`, whose stage failed on this + /// leader: hold it for a restage, or halt once it cannot commit here. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn restage_or_halt( + &mut self, + txn_id: TxnId, + ) { + let Some((superseded, stage_error)) = self.pending.get_mut(&txn_id).map(|pending| { + pending.verdict_deadline = None; + ( + pending.superseded, + pending.stage_error.clone().unwrap_or_default(), + ) + }) else { + return; + }; + if superseded { + self.halt_apply( + txn_id, + HaltReason::LocalStageFailed, + HaltStep::Stage, + format!( + "COMMIT verdict for a txn whose collection this leader found superseded: \ + {stage_error}" + ), + ); + return; + } + let done = self + .restages + .get(&txn_id) + .map_or(0, |restage| restage.attempts); + if done >= self.config.restage_attempts { + self.restages.remove(&txn_id); + self.halt_apply( + txn_id, + HaltReason::LocalStageFailed, + HaltStep::Stage, + format!( + "COMMIT verdict for a txn this leader failed to stage in {done} restages: \ + {stage_error}" + ), + ); + return; + } + if let Some(pending) = self.pending.get_mut(&txn_id) { + pending.commit_state = CommitState::AwaitingRestage; + pending.awaiting = None; + pending.stage_error = None; + } + let backoff = self.config.restage_backoff(done); + // no-determinism: the restage backoff only delays a retry of a decided COMMIT verdict; it never changes what a replica applies. + let due = Instant::now() + backoff; + self.restages.insert( + txn_id, + Restage { + attempts: done, + due: Some(due), + }, + ); + warn!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + restages_done = done, + backoff_ms = u64::try_from(backoff.as_millis()).unwrap_or(u64::MAX), + error = %stage_error, + "calvin: COMMIT verdict for a txn this leader failed to stage; staging it again \ + after the backoff" + ); + // Clear what the failed stage left on the core. The restage reaches + // the same core after it, in dispatch order. + self.discard_staged(txn_id); + } + + /// Stage again, in sequencer order, every txn whose restage backoff + /// passed. Runs on the leader only, and not once the scheduler halted. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn restage_due(&mut self) { + if self.restages.is_empty() || !self.role.is_leader() || self.is_apply_halted() { + return; + } + // no-determinism: the restage backoff only delays a retry of a decided COMMIT verdict. + let now = Instant::now(); + let mut due = Vec::new(); + for (txn_id, restage) in &self.restages { + let Some(at) = restage.due else { + continue; + }; + // A lower txn's restage goes first. A discard that waits for + // capacity holds its restage back, so the stage never overtakes it. + if at > now || self.has_deferred_for(*txn_id) { + break; + } + due.push(*txn_id); + } + for txn_id in due { + self.restage(txn_id); + if self.is_apply_halted() || !self.role.is_leader() { + return; + } + } + } + + /// When the run loop next wakes for a restage: the backoff end of the + /// lowest txn that waits for its restage. `None` when no restage can + /// run, or when one waits for capacity, which wakes the loop itself. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn next_restage_at( + &self, + ) -> Option { + if !self.role.is_leader() || self.is_apply_halted() { + return None; + } + let (txn_id, at) = self + .restages + .iter() + .find_map(|(txn_id, restage)| restage.due.map(|at| (*txn_id, at)))?; + (!self.has_deferred_for(txn_id)).then_some(at) + } + + /// Forget the restage and the held reads of `txn_id`, which finished. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn forget_held_state( + &mut self, + txn_id: TxnId, + ) { + self.restages.remove(&txn_id); + self.held_reads.remove(&txn_id); + } + + /// Stage `txn_id` again through the grant path. + fn restage(&mut self, txn_id: TxnId) { + let awaits = self + .pending + .get(&txn_id) + .is_some_and(|pending| pending.commit_state == CommitState::AwaitingRestage); + let Some(restage) = self.restages.get_mut(&txn_id) else { + return; + }; + if !awaits { + restage.due = None; + return; + } + restage.attempts = restage.attempts.saturating_add(1); + restage.due = None; + let attempt = restage.attempts; + let Some(pending) = self.pending.remove(&txn_id) else { + return; + }; + self.metrics.record_restage(); + self.shared + .calvin + .counters + .stage_restages + .fetch_add(1, Ordering::Relaxed); + info!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + attempt, + "calvin: staging a committed txn again" + ); + let PendingTxn { + txn, + lock_owner, + gates, + .. + } = pending; + // The old gates stay held until the restage holds its own, so a + // purge of a named collection never runs between the two. + self.route_granted(txn, txn_id, lock_owner); + drop(gates); + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::time::Duration; + + use nodedb_cluster::calvin::types::SchedulerInput; + use nodedb_cluster::calvin::{CalvinCompletionRegistry, VerdictOutcome}; + use nodedb_physical::physical_plan::PhysicalPlan; + use nodedb_physical::physical_plan::meta::MetaOp; + + use super::*; + use crate::bridge::dispatch::CoreChannelDataSide; + use crate::bridge::envelope::{ErrorCode, StageVote, Status}; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ + lead_data_group, step_down_from_data_group, + }; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler_with_data_side, error_response, make_local_write_txn, staged_response, + test_coll_vshard, + }; + + /// A data-group leader on the `test_coll` vShard, sharing `registry`. + fn leader( + registry: &Arc, + ) -> (Scheduler, tempfile::TempDir, CoreChannelDataSide) { + let (mut scheduler, dir, data_side) = + build_test_scheduler_with_data_side(test_coll_vshard(), Arc::clone(registry)); + lead_data_group(&mut scheduler); + (scheduler, dir, data_side) + } + + /// The Calvin plans on the Data Plane side, in arrival order. Drains + /// the request ring. + fn requests(data_side: &mut CoreChannelDataSide) -> Vec { + let mut plans = Vec::new(); + while let Ok(request) = data_side.request_rx.try_pop() { + plans.push(request.inner.plan); + } + plans + } + + fn is_stage(plan: &PhysicalPlan, txn_id: TxnId) -> bool { + matches!( + plan, + PhysicalPlan::Meta(MetaOp::CalvinExecuteStatic { epoch, position, .. }) + if *epoch == txn_id.epoch && *position == txn_id.position + ) + } + + fn is_drop(plan: &PhysicalPlan, txn_id: TxnId) -> bool { + *plan + == PhysicalPlan::Meta(MetaOp::CalvinDrop { + epoch: txn_id.epoch, + position: txn_id.position, + }) + } + + fn stage_failed() -> crate::bridge::envelope::Response { + error_response(ErrorCode::Internal { + detail: "stage failed on this core".to_string(), + }) + } + + /// Store a COMMIT verdict for `txn_id`, as the sequencer log applies it. + fn commit_verdict(registry: &CalvinCompletionRegistry, txn_id: TxnId) { + registry.note_verdict( + nodedb_cluster::calvin::TxnId::new(txn_id.epoch, txn_id.position), + VerdictOutcome::Commit, + ); + } + + /// End the backoff of `txn_id`'s next restage now. + fn end_backoff(scheduler: &mut Scheduler, txn_id: TxnId) { + let restage = scheduler + .restages + .get_mut(&txn_id) + .expect("the txn waits for a restage"); + // no-determinism: test-only backoff end in the past. + restage.due = Some(Instant::now() - Duration::from_millis(1)); + } + + fn restages(scheduler: &Scheduler) -> u64 { + scheduler + .shared + .calvin + .counters + .stage_restages + .load(Ordering::Relaxed) + } + + /// A leader whose stage of a committed write slice failed: it stages the + /// slice again after the backoff, fails once more, then stages it and + /// commits. The resolve follows the normal commit path. + #[tokio::test] + async fn a_restage_that_succeeds_on_the_second_attempt_commits() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, mut data_side) = leader(®istry); + scheduler.config.restage_backoff_ms = 60_000; + let txn_id = TxnId::new(20, 0); + scheduler + .process_scheduler_input(SchedulerInput::Txn(Box::new(make_local_write_txn(20, 0)))); + assert!(requests(&mut data_side).iter().any(|p| is_stage(p, txn_id))); + + scheduler.resolve_staged_commit(txn_id, &stage_failed()); + commit_verdict(®istry, txn_id); + scheduler.resume_on_verdict(txn_id, true); + + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingRestage) + ); + assert!( + requests(&mut data_side).iter().any(|p| is_drop(p, txn_id)), + "the failed stage's state is discarded" + ); + assert!(!scheduler.is_apply_halted()); + + scheduler.restage_due(); + assert!( + requests(&mut data_side).is_empty(), + "nothing stages before the backoff ends" + ); + assert!(scheduler.next_restage_at().is_some()); + + end_backoff(&mut scheduler, txn_id); + scheduler.restage_due(); + assert!(requests(&mut data_side).iter().any(|p| is_stage(p, txn_id))); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Staged) + ); + assert_eq!(restages(&scheduler), 1); + + // The second stage fails too. The known verdict holds it for the + // next restage at once. + scheduler.resolve_staged_commit(txn_id, &stage_failed()); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingRestage) + ); + assert_eq!(scheduler.restages.get(&txn_id).map(|r| r.attempts), Some(1)); + requests(&mut data_side); + + end_backoff(&mut scheduler, txn_id); + scheduler.restage_due(); + assert!(requests(&mut data_side).iter().any(|p| is_stage(p, txn_id))); + assert_eq!(restages(&scheduler), 2); + + scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingRedoResolve) + ); + assert!( + requests(&mut data_side).iter().any(|p| matches!( + p, + PhysicalPlan::Meta(MetaOp::CalvinResolve { + epoch: 20, + position: 0 + }) + )), + "the committed slice resolves its redo" + ); + assert!(!scheduler.is_apply_halted()); + assert!(!scheduler.applied.is_applied(20, 0)); + } + + /// Once its restages run out, a leader that still cannot stage a + /// committed txn halts. The txn stays parked with its locks, unapplied. + #[tokio::test] + async fn a_spent_restage_bound_halts_unapplied() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, mut data_side) = leader(®istry); + scheduler.config.restage_attempts = 1; + let txn_id = TxnId::new(21, 0); + scheduler + .process_scheduler_input(SchedulerInput::Txn(Box::new(make_local_write_txn(21, 0)))); + commit_verdict(®istry, txn_id); + scheduler.resolve_staged_commit(txn_id, &stage_failed()); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingRestage) + ); + + end_backoff(&mut scheduler, txn_id); + scheduler.restage_due(); + requests(&mut data_side); + scheduler.resolve_staged_commit(txn_id, &stage_failed()); + + assert_eq!( + scheduler.apply_halt().map(|h| h.reason), + Some(HaltReason::LocalStageFailed) + ); + let pending = scheduler + .pending + .get(&txn_id) + .expect("the txn stays pending"); + assert_eq!(pending.commit_state, CommitState::AwaitingVerdict); + assert_eq!(pending.verdict_deadline, None); + assert!(scheduler.restages.is_empty()); + assert_eq!(restages(&scheduler), 1); + assert!(!scheduler.applied.is_applied(21, 0)); + assert!( + !requests(&mut data_side) + .iter() + .any(|p| matches!(p, PhysicalPlan::Meta(MetaOp::CalvinResolve { .. }))), + "no resolve reaches the Data Plane" + ); + } + + /// A committed txn whose collection this leader found superseded cannot + /// commit here. It halts at once, with no restage. + #[tokio::test] + async fn a_superseded_collection_halts_at_once() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, mut data_side) = leader(®istry); + let txn_id = TxnId::new(22, 0); + scheduler + .process_scheduler_input(SchedulerInput::Txn(Box::new(make_local_write_txn(22, 0)))); + requests(&mut data_side); + if let Some(pending) = scheduler.pending.get_mut(&txn_id) { + pending.superseded = true; + } + commit_verdict(®istry, txn_id); + + scheduler.resolve_staged_commit( + txn_id, + &staged_response(Status::Ok, Some(StageVote::Commit)), + ); + + assert_eq!( + scheduler.apply_halt().map(|h| h.reason), + Some(HaltReason::LocalStageFailed) + ); + assert!(scheduler.restages.is_empty()); + assert_eq!(restages(&scheduler), 0); + assert!( + requests(&mut data_side).is_empty(), + "nothing is discarded or staged" + ); + } + + /// An abort verdict for a txn this leader failed to stage drops it. No + /// restage follows. + #[tokio::test] + async fn an_abort_verdict_drops_without_restaging() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, mut data_side) = leader(®istry); + let txn_id = TxnId::new(23, 0); + scheduler + .process_scheduler_input(SchedulerInput::Txn(Box::new(make_local_write_txn(23, 0)))); + requests(&mut data_side); + scheduler.resolve_staged_commit(txn_id, &stage_failed()); + + scheduler.resume_on_verdict(txn_id, false); + + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingDrop) + ); + assert!(requests(&mut data_side).iter().any(|p| is_drop(p, txn_id))); + assert!(scheduler.restages.is_empty()); + assert_eq!(scheduler.next_restage_at(), None); + assert_eq!(restages(&scheduler), 0); + assert!(!scheduler.is_apply_halted()); + } + + /// A txn whose plans this leader rejected restages through the grant + /// path. The plans reject again, which spends the bound and halts. + #[tokio::test] + async fn a_rejected_plan_restages_through_the_grant_path() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, _data_side) = leader(®istry); + scheduler.config.restage_attempts = 1; + let txn_id = TxnId::new(24, 0); + let mut txn = make_local_write_txn(24, 0); + txn.tx_class.plans = vec![0xc1]; + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(txn))); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingVerdict) + ); + + commit_verdict(®istry, txn_id); + scheduler.resume_on_verdict(txn_id, true); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingRestage) + ); + + end_backoff(&mut scheduler, txn_id); + scheduler.restage_due(); + + assert_eq!(restages(&scheduler), 1); + assert_eq!( + scheduler.apply_halt().map(|h| h.reason), + Some(HaltReason::LocalStageFailed) + ); + assert!( + scheduler.pending.contains_key(&txn_id), + "the rejected restage parks the txn again" + ); + } + + /// A demoted leader holds a txn awaiting its restage `Following`, and + /// forgets the restage. + #[tokio::test] + async fn demotion_forgets_a_waiting_restage() { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, _dir, _data_side) = leader(®istry); + let txn_id = TxnId::new(25, 0); + scheduler + .process_scheduler_input(SchedulerInput::Txn(Box::new(make_local_write_txn(25, 0)))); + commit_verdict(®istry, txn_id); + scheduler.resolve_staged_commit(txn_id, &stage_failed()); + assert!(scheduler.restages.contains_key(&txn_id)); + + step_down_from_data_group(&scheduler); + scheduler.refresh_role(); + + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Following) + ); + assert!(scheduler.restages.is_empty()); + assert_eq!(scheduler.next_restage_at(), None); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/role.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/role.rs index d702c9b28..e3d085ec7 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/role.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/role.rs @@ -16,9 +16,9 @@ //! order. A txn whose verdict is already COMMIT proposes its vote again, //! which changes nothing: the first vote of the vShard counts. //! - Demotion discards the staged state of every txn the leader drove, -//! drops its owed votes, and holds the txn `Following`. A redo this node -//! proposed can still commit in the new term: the applied ledger installs -//! one copy per position. +//! drops its owed votes and its restages, and holds the txn `Following`. +//! A redo this node proposed can still commit in the new term: the +//! applied ledger installs one copy per position. //! //! [`CommitState::Following`]: super::super::types::CommitState::Following @@ -182,12 +182,16 @@ impl Scheduler { for txn_id in driven { self.release_to_following(txn_id); } + // Only a leader restages. A later term counts its own restages. + self.restages.clear(); // A barrier waits only on the leader. Its txn holds its locks and - // follows the log like any other. + // follows the log like any other. It keeps the reads it received, + // so a later barrier of it starts from them. let barriers: Vec = self.dependent_barrier.keys().copied().collect(); for txn_id in barriers { if let Some(barrier) = self.dependent_barrier.remove(&txn_id) { self.follow(barrier.txn, txn_id, barrier.lock_owner); + self.hold_reads(txn_id, barrier.received); } } } @@ -238,7 +242,8 @@ fn leader_drives(state: CommitState) -> bool { | CommitState::AwaitingVerdict | CommitState::AwaitingResolveTurn | CommitState::AwaitingRedoResolve - | CommitState::AwaitingRedoApply { .. } => true, + | CommitState::AwaitingRedoApply { .. } + | CommitState::AwaitingRestage => true, // A drop ends the txn with no log entry whatever the role. CommitState::AwaitingDrop | CommitState::Following => false, } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs index b3a0767ea..d3ca26bd6 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs @@ -57,6 +57,7 @@ impl Scheduler { self.drain_inbox(); self.check_dependent_barrier_timeouts(); self.check_awaiting_verdict_stalls(); + self.restage_due(); self.resume_metadata_hold(); // Open only once the inputs a closed gate held are processed. A @@ -67,6 +68,8 @@ impl Scheduler { catch_up_resume = false; stall_tick.reset_immediately(); } + // The backoff end of the next restage, when one waits. + let restage_at = self.next_restage_at().map(tokio::time::Instant::from_std); tokio::select! { biased; @@ -130,6 +133,13 @@ impl Scheduler { // The next loop pass re-checks the held txn's floor. } + _ = tokio::time::sleep_until( + restage_at.unwrap_or_else(tokio::time::Instant::now) + ), if restage_at.is_some() => { + // The next loop pass stages again every txn whose + // restage backoff ended. + } + _ = tokio::time::sleep(ROLE_POLL), if self.role.awaits_term_start() => { // The next loop pass checks whether the term's no-op // applied, and stages the held txns once it has. diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs index de8efb90c..4f8c4df2c 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs @@ -102,6 +102,15 @@ pub struct Scheduler { /// `BTreeMap` for determinism. pub(in crate::control::cluster::calvin::scheduler::driver::core) dependent_barrier: BTreeMap, + /// Passive read results of dependent txns in `pending` that wait at no + /// barrier. A later barrier of the txn starts from them. Each entry + /// leaves when its txn completes. + pub(in crate::control::cluster::calvin::scheduler::driver::core) held_reads: + BTreeMap, + /// Committed txns whose stage failed on this leader, with their + /// restage count and backoff. See [`super::restage`]. + pub(in crate::control::cluster::calvin::scheduler::driver::core) restages: + BTreeMap, /// Channel receiving `CalvinReadResult` Raft apply events from the /// per-vshard data Raft apply loop. Bounded. pub(in crate::control::cluster::calvin::scheduler::driver::core) read_result_rx: @@ -296,6 +305,8 @@ impl Scheduler { inbox, blocked: BTreeMap::new(), dependent_barrier: BTreeMap::new(), + held_reads: BTreeMap::new(), + restages: BTreeMap::new(), read_result_rx, applied: AppliedGate::new(fully_applied_epoch, applied_tail), cut_floors: Default::default(), diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs index 3ecc4d098..4fdcda2d9 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs @@ -66,8 +66,9 @@ pub(super) struct PendingTxn { /// Error text of a stage response that was not `Ok` on this leader. /// /// `Some` when this leader never staged the txn. An abort verdict drops - /// it as usual. A COMMIT verdict halts the scheduler, because the txn - /// cannot apply here while its peers apply it. + /// it as usual. A COMMIT verdict restages it, a bounded number of times. + /// A superseded collection or a spent bound halts the scheduler, because + /// the txn cannot apply here while its peers apply it. pub stage_error: Option, /// What this vShard's slice writes, derived once from its local plans. pub scope: SliceScope, @@ -211,6 +212,10 @@ pub(in crate::control::cluster::calvin::scheduler::driver) enum CommitState { /// verdict, or a COMMIT verdict for a slice with no write. The txn /// completes once the drop answers. AwaitingDrop, + /// The verdict is COMMIT, and this leader's stage failed. Its staged + /// state is discarded. The txn holds its locks and stages again once + /// its backoff passes. + AwaitingRestage, } /// A transaction that is blocked on lock acquisition. diff --git a/nodedb/src/control/cluster/calvin/scheduler/metrics.rs b/nodedb/src/control/cluster/calvin/scheduler/metrics.rs index b89b1e766..66df309e9 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/metrics.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/metrics.rs @@ -80,6 +80,8 @@ pub struct SchedulerMetrics { /// applied, by kind. Indexes are the constants in /// [`sequencer_propose_kind`]. pub sequencer_propose_retry_counts: [AtomicU64; sequencer_propose_kind::LABELS.len()], + /// Committed txns this leader staged again after its own stage failed. + pub restage_count: AtomicU64, } /// Reason codes for `nodedb_calvin_infra_abort_total`. @@ -211,6 +213,11 @@ impl SchedulerMetrics { self.catch_up_log_compacted.fetch_add(1, Ordering::Relaxed); } + /// Record that the leader staged a committed txn again. + pub fn record_restage(&self) { + self.restage_count.fetch_add(1, Ordering::Relaxed); + } + /// Record that the dispatcher refused a scheduler dispatch at capacity. pub fn record_dispatch_deferred(&self) { self.dispatch_deferred_count.fetch_add(1, Ordering::Relaxed); @@ -444,6 +451,7 @@ impl Default for SchedulerMetrics { apply_halted: AtomicU64::new(0), apply_halt_reason: AtomicU64::new(0), sequencer_propose_retry_counts: std::array::from_fn(|_| AtomicU64::new(0)), + restage_count: AtomicU64::new(0), } } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/metrics_flow.rs b/nodedb/src/control/cluster/calvin/scheduler/metrics_flow.rs index 2117749e3..5412b564e 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/metrics_flow.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/metrics_flow.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 //! Prometheus rendering of the scheduler's flow metrics: deferred dispatch, -//! the intake gate, the apply halt, and sequencer propose retries. +//! the intake gate, the apply halt, sequencer propose retries, and restages. use std::fmt::Write as _; use std::sync::atomic::Ordering; @@ -106,6 +106,18 @@ impl SchedulerMetrics { self.sequencer_propose_retry_counts[i].load(Ordering::Relaxed) ); } + + let _ = writeln!( + out, + "# HELP nodedb_calvin_restage_total \ + Committed txns the leader staged again after its own stage failed." + ); + let _ = writeln!(out, "# TYPE nodedb_calvin_restage_total counter"); + let _ = writeln!( + out, + "nodedb_calvin_restage_total{{{label}}} {}", + self.restage_count.load(Ordering::Relaxed) + ); } } @@ -129,4 +141,12 @@ mod tests { "nodedb_calvin_sequencer_propose_retry_total{vshard=\"3\",kind=\"completion_ack\"} 1" )); } + + #[test] + fn restage_counter_renders() { + let m = SchedulerMetrics::new(); + m.record_restage(); + let out = m.render_prometheus(4); + assert!(out.contains("nodedb_calvin_restage_total{vshard=\"4\"} 1")); + } } diff --git a/nodedb/src/control/cluster/start_raft/observability.rs b/nodedb/src/control/cluster/start_raft/observability.rs index f17d0f41d..53357fe5e 100644 --- a/nodedb/src/control/cluster/start_raft/observability.rs +++ b/nodedb/src/control/cluster/start_raft/observability.rs @@ -175,6 +175,13 @@ fn publish_raft_status( if shared.raft_read_gate.set(gate).is_err() { tracing::warn!("raft_read_gate already set — start_raft appears to have run twice"); } + if shared + .multi_raft + .set(raft_loop.multi_raft_handle()) + .is_err() + { + tracing::warn!("multi_raft already set — start_raft appears to have run twice"); + } } /// Renew this node's authorization lease with the metadata leader. Its diff --git a/nodedb/src/control/state/calvin_counters.rs b/nodedb/src/control/state/calvin_counters.rs index 03d0552e0..acdc0fb8a 100644 --- a/nodedb/src/control/state/calvin_counters.rs +++ b/nodedb/src/control/state/calvin_counters.rs @@ -29,4 +29,7 @@ pub struct CalvinCounters { /// installed nothing: their position was applied, or another copy held /// its claim. pub redo_copies_skipped: Arc, + /// Count of restages this node's leaders dispatched: a committed Calvin + /// slice staged again after the leader's own stage failed. + pub stage_restages: Arc, } diff --git a/nodedb/src/control/state/calvin_local.rs b/nodedb/src/control/state/calvin_local.rs index b7fd2525e..495106d2e 100644 --- a/nodedb/src/control/state/calvin_local.rs +++ b/nodedb/src/control/state/calvin_local.rs @@ -108,6 +108,7 @@ impl CalvinLocalState { commits_flushed: Arc::new(AtomicU64::new(0)), commits_dropped: Arc::new(AtomicU64::new(0)), redo_copies_skipped: Arc::new(AtomicU64::new(0)), + stage_restages: Arc::new(AtomicU64::new(0)), }, apply_results: CalvinApplySidecar::default(), lock_managers: Arc::new(Mutex::new(BTreeMap::new())), diff --git a/nodedb/src/control/state/fields.rs b/nodedb/src/control/state/fields.rs index f1cbdb6e2..29891b91a 100644 --- a/nodedb/src/control/state/fields.rs +++ b/nodedb/src/control/state/fields.rs @@ -162,6 +162,8 @@ pub struct SharedState { /// linearizable read, replica freshness for a bounded-staleness one. /// Set by `start_raft`. pub raft_read_gate: OnceLock>, + /// This node's Raft groups, for a leadership move. Set by `start_raft`. + pub multi_raft: OnceLock>>, /// This node's cluster-epoch state. Set once by `start_raft`. pub cluster_epoch: OnceLock>, /// Query Raft group statuses for observability. Set by `start_raft`. diff --git a/nodedb/src/control/state/init.rs b/nodedb/src/control/state/init.rs index 006096ab6..f1fbf5f35 100644 --- a/nodedb/src/control/state/init.rs +++ b/nodedb/src/control/state/init.rs @@ -155,6 +155,7 @@ impl SharedState { raft_applied_index_sink: std::sync::OnceLock::new(), raft_apply_gates: std::sync::OnceLock::new(), raft_read_gate: std::sync::OnceLock::new(), + multi_raft: std::sync::OnceLock::new(), cluster_epoch: std::sync::OnceLock::new(), raft_status_fn: std::sync::OnceLock::new(), cluster_observer: std::sync::OnceLock::new(), diff --git a/nodedb/src/control/state/init_prod/open.rs b/nodedb/src/control/state/init_prod/open.rs index 9084aec7b..256000b9c 100644 --- a/nodedb/src/control/state/init_prod/open.rs +++ b/nodedb/src/control/state/init_prod/open.rs @@ -226,6 +226,7 @@ impl SharedState { raft_applied_index_sink: std::sync::OnceLock::new(), raft_apply_gates: std::sync::OnceLock::new(), raft_read_gate: std::sync::OnceLock::new(), + multi_raft: std::sync::OnceLock::new(), cluster_epoch: std::sync::OnceLock::new(), raft_status_fn: std::sync::OnceLock::new(), cluster_observer: std::sync::OnceLock::new(), From 38cf52dcfb119f06edaba11c1fe86f3cdbe7e943 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sat, 10 Oct 2026 12:50:03 +0800 Subject: [PATCH 12/19] test(cluster): cover Calvin leadership moves with an uncommitted entry Adds harness helpers to move and wait on data-group leadership, and cluster cases for a split leader and an old-term redo copy. --- .../cases/calvin_old_term_redo_copy.rs | 307 ++++++++++++++++++ .../common_suite/cases/calvin_split_leader.rs | 195 +++++++++++ .../tests/common_suite/cases/mod.rs | 2 + .../src/cluster_harness/cluster/leadership.rs | 191 +++++++++++ .../src/cluster_harness/cluster/mod.rs | 5 +- .../src/cluster_harness/mod.rs | 2 +- .../src/cluster_harness/node/leadership.rs | 39 +++ .../src/cluster_harness/node/mod.rs | 1 + 8 files changed, 740 insertions(+), 2 deletions(-) create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_old_term_redo_copy.rs create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_split_leader.rs create mode 100644 nodedb-test-support/src/cluster_harness/cluster/leadership.rs create mode 100644 nodedb-test-support/src/cluster_harness/node/leadership.rs diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_old_term_redo_copy.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_old_term_redo_copy.rs new file mode 100644 index 000000000..0b0d8fe7a --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_old_term_redo_copy.rs @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A Calvin slice's redo that its leader proposed, and that commits only in +//! the next term under a new leader, installs once. The new leader stages +//! nothing for that position. +//! +//! 1. A 3-node cluster holds `accts`, whose `balance` is the materialized +//! sum of `entries.amount`, and `entries`, in distinct data groups. +//! `acc-0` starts at 100. Every data group is led by its preferred +//! leader, so the leader balance moves nothing by itself. +//! 2. The fail gate `calvin::before_redo_propose::` holds the +//! entries slice after its vote and resolve, on the leader `L` of the +//! entries group `G`. The balance slice installs meanwhile. +//! 3. The test arms `raft::hold_commit::node::group`, then releases +//! the redo. `L` proposes it, the entry replicates to every replica, and +//! `L` commits nothing. +//! 4. The test moves `G`'s leadership to another replica `N`. The entry of +//! the old term commits with `N`'s first entry of the new term. `N` +//! opens its stage gate only once that entry applied, so it finds the +//! position applied and stages nothing. +//! 5. The client's COMMIT succeeds once. No node resolved a commit or +//! restaged after the hold, no node applied a second copy, each replica +//! of `G` installed the slice once, and `acc-0` reads 125, the value of +//! a single install. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::sync::atomic::Ordering; +use std::time::Duration; + +use nodedb_types::fail_point::{FailAction, FailGuard}; + +use super::calvin_multishard_fixture::{Fixture, tags}; +use super::calvin_replica_content::{ + first_column, run_retrying, sum_ddl, vshard_of, wait_replicas_agree, +}; +use crate::common::cluster_harness::shared_steps::{group_members, group_status, name_where}; +use crate::common::cluster_harness::{ + GroupLeader, TestCluster, TestClusterNode, wait_for, wait_for_async, +}; + +/// How long each step of the hold gets. +const STEP_DEADLINE: Duration = Duration::from_secs(30); + +/// Pause between two polls of a step. +const STEP_POLL: Duration = Duration::from_millis(20); + +/// One node's Calvin counters. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Counters { + /// Committed slices this node's leaders resolved. + commits: u64, + /// Restages this node's leaders dispatched. + restages: u64, + /// Stamped redo copies this node applied that installed nothing. + skipped_copies: u64, + /// Stamped redo installs on this node. + installs: u64, +} + +impl Counters { + fn of(node: &TestClusterNode) -> Self { + let counters = &node.shared.calvin.counters; + Self { + commits: counters.commits_flushed.load(Ordering::Relaxed), + restages: counters.stage_restages.load(Ordering::Relaxed), + skipped_copies: counters.redo_copies_skipped.load(Ordering::Relaxed), + installs: counters.write_versions_recorded.load(Ordering::Relaxed), + } + } +} + +/// The served balance of `acc-0`, `None` while no leader serves it. +async fn balance(node: &TestClusterNode, accts: &str) -> Option { + let sql = format!("SELECT balance FROM {accts} WHERE id = 'acc-0'"); + let rows = node.client.simple_query(&sql).await.ok()?; + rows.into_iter().find_map(|msg| match msg { + tokio_postgres::SimpleQueryMessage::Row(row) => row.get(0).map(str::to_owned), + _ => None, + }) +} + +/// The index in `cluster.nodes` of node `node_id`. +fn index_of(cluster: &TestCluster, node_id: u64) -> usize { + cluster + .nodes + .iter() + .position(|node| node.node_id == node_id) + .unwrap_or_else(|| panic!("node {node_id} is live")) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn an_old_term_redo_copy_committed_by_the_new_leader_installs_once() { + let fx = Fixture::from_cluster(TestCluster::spawn_three().await.expect("3-node cluster")).await; + let probe = &fx.cluster.nodes[0]; + let entries = name_where("old_term_entries", |name| { + probe.group_id_for_collection(name).is_some() + }); + let group = probe + .group_id_for_collection(&entries) + .expect("the entries vShard maps to a group"); + let accts = name_where("old_term_accts", |name| { + vshard_of(name) != vshard_of(&entries) + && probe + .group_id_for_collection(name) + .is_some_and(|other| other != group) + }); + let [accts_ddl, entries_ddl, sum] = sum_ddl(&entries, &accts); + fx.create(&[accts_ddl, entries_ddl]).await; + fx.cluster + .exec_ddl_on_any_leader(&sum) + .await + .unwrap_or_else(|e| panic!("{sum}: {e}")); + for collection in [&accts, &entries] { + fx.wait_group_mounted(collection).await; + } + run_retrying( + &fx.cluster.nodes[0].client, + "seed the account", + &format!("INSERT INTO {accts} (id, owner, balance) VALUES ('acc-0', 'seed', '100')"), + ) + .await; + fx.converge().await; + fx.cluster.wait_for_preferred_leaders(STEP_DEADLINE).await; + + let mut session = fx.raw().await; + let txn = format!( + "BEGIN; \ + INSERT INTO {entries} (id, account_id, amount) VALUES ('e1', 'acc-0', '25'); \ + COMMIT" + ); + let gate_dir = tempfile::tempdir().expect("gate directory"); + let propose_release = gate_dir.path().join("propose"); + let propose_parked = gate_dir.path().join("propose.parked"); + let commit_release = gate_dir.path().join("commit"); + let (committed, before) = { + let _propose_gate = FailGuard::install( + &format!("calvin::before_redo_propose::{entries}"), + FailAction::WaitForFile(propose_release.clone()), + ); + let commit = tags(&mut session, &txn); + let move_leader = async { + wait_for( + "the entries leader holds the slice's redo after its resolve", + STEP_DEADLINE, + STEP_POLL, + || propose_parked.exists(), + ) + .await; + let old_leader = fx + .cluster + .wait_for_group_leader(group, GroupLeader::Any, STEP_DEADLINE) + .await; + let old = index_of(&fx.cluster, old_leader); + // The balance slice installs on every replica while the entries + // slice is held. Every count below then belongs to the held slice. + let coordinator = fx.coordinator(); + let accts = accts.as_str(); + wait_for_async( + "the balance slice installs", + STEP_DEADLINE, + STEP_POLL, + || async move { balance(coordinator, accts).await.as_deref() == Some("125") }, + ) + .await; + fx.converge().await; + let before: Vec = fx.cluster.nodes.iter().map(Counters::of).collect(); + let held_status = group_status(&fx.cluster.nodes[old], group) + .expect("the old leader hosts the entries group"); + + let _commit_gate = FailGuard::install( + &format!("raft::hold_commit::node{old_leader}::group{group}"), + FailAction::WaitForFile(commit_release.clone()), + ); + std::fs::write(&propose_release, b"release").expect("release the held redo"); + let new_leader = group_members(&fx.cluster.nodes[0], group) + .into_iter() + .find(|member| *member != old_leader) + .expect("the entries group has a replica besides its leader"); + let new = index_of(&fx.cluster, new_leader); + let mut redo_index = 0; + wait_for( + "the old leader appends the redo, and the new leader holds it, uncommitted", + STEP_DEADLINE, + STEP_POLL, + || { + let (Some(old_status), Some(new_status)) = ( + group_status(&fx.cluster.nodes[old], group), + group_status(&fx.cluster.nodes[new], group), + ) else { + return false; + }; + redo_index = old_status.last_log_index; + old_status.last_log_index > held_status.last_log_index + && old_status.commit_index < old_status.last_log_index + && new_status.last_log_index >= old_status.last_log_index + }, + ) + .await; + + fx.cluster.transfer_leadership(group, new_leader).await; + let new_status = group_status(&fx.cluster.nodes[new], group) + .expect("the new leader hosts the entries group"); + assert!( + new_status.term > held_status.term, + "the new leader leads a later term" + ); + std::fs::write(&commit_release, b"release").expect("release the commit gate"); + wait_for( + "the redo of the old term commits under the new leader", + STEP_DEADLINE, + STEP_POLL, + || { + group_status(&fx.cluster.nodes[new], group) + .is_some_and(|status| status.commit_index >= redo_index) + }, + ) + .await; + before + }; + tokio::join!(commit, move_leader) + }; + + assert_eq!( + committed + .iter() + .filter(|tag| tag.as_str() == "COMMIT") + .count(), + 1, + "the client's COMMIT answers once, found tags {committed:?}" + ); + assert_eq!( + committed.last().map(String::as_str), + Some("COMMIT"), + "the transaction ends committed, found tags {committed:?}" + ); + + fx.converge().await; + let replicas: Vec = fx + .cluster + .nodes + .iter() + .map(|node| node.replicates_data_group(group)) + .collect(); + wait_for( + "every replica of the entries group installs the slice", + STEP_DEADLINE, + STEP_POLL, + || { + fx.cluster + .nodes + .iter() + .zip(&before) + .zip(&replicas) + .all(|((node, before), replica)| { + !replica || Counters::of(node).installs > before.installs + }) + }, + ) + .await; + for ((node, before), replica) in fx.cluster.nodes.iter().zip(&before).zip(&replicas) { + let after = Counters::of(node); + assert_eq!( + after.installs - before.installs, + u64::from(*replica), + "node {} installs the slice once when it replicates the entries group", + node.node_id + ); + assert_eq!( + after.skipped_copies, before.skipped_copies, + "node {} applies no second copy of the redo", + node.node_id + ); + assert_eq!( + after.commits, before.commits, + "node {} resolves no commit after the hold: no leader staged the position again", + node.node_id + ); + assert_eq!( + after.restages, before.restages, + "node {} restages nothing", + node.node_id + ); + } + + let collections = [accts.as_str(), entries.as_str()]; + wait_replicas_agree(&fx.cluster, &collections).await; + let client = &fx.coordinator().client; + assert_eq!( + first_column( + client, + &format!("SELECT balance FROM {accts} WHERE id = 'acc-0'") + ) + .await, + vec!["125".to_owned()], + "100 + 25 lands once; a second install reads 150" + ); + assert_eq!( + first_column(client, &format!("SELECT id FROM {entries}")).await, + vec!["e1".to_owned()], + "the entries slice wrote its row once" + ); + + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_split_leader.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_split_leader.rs new file mode 100644 index 000000000..a0a12b8fe --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_split_leader.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Concurrent cross-shard Calvin transactions commit exactly once while the +//! data-group leadership of their vShards moves between nodes. +//! +//! 1. A 3-node cluster holds `accts`, whose `balance` is the materialized +//! sum of `entries.amount`, and `entries`, on distinct vShards. Each +//! account starts at 0. +//! 2. Two sessions on two nodes run transactions at once. Each inserts one +//! entry, which moves its account's balance on the other vShard. +//! 3. Meanwhile the test moves the leadership of each participant data +//! group to another replica, several times. A moved leader drops what it +//! staged, and the new leader stages its held transactions again. +//! 4. After quiesce, every entry a COMMIT answered for is stored once, +//! every replica holds the same content, and each balance equals both +//! the sum of its entries and the sum the client committed. + +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use super::calvin_multishard_fixture::Fixture; +use super::calvin_replica_content::{ + first_column, names_on_distinct_vshards, run_retrying, strict_session, sum_ddl, + wait_replicas_agree, +}; +use crate::common::cluster_harness::shared_steps::{group_members, group_of}; +use crate::common::cluster_harness::{GroupLeader, TestCluster}; + +/// Transactions each session runs. +const ROUNDS: usize = 12; + +/// Sessions that run transactions at once, one per node. +const SESSIONS: usize = 2; + +/// Accounts in `accts`, `acc-0` to `acc-{ACCOUNTS - 1}`. +const ACCOUNTS: usize = 3; + +/// Leadership moves of each participant data group. +const MOVES: usize = 6; + +/// Pause between two rounds of moves, so transactions run under each +/// leader. +const MOVE_PAUSE: Duration = Duration::from_millis(250); + +/// How long a group gets to show a leader before a move. +const LEADER_WAIT: Duration = Duration::from_secs(30); + +/// The account transaction `k` of session `session` books to. +fn account(session: usize, k: usize) -> usize { + (session + k) % ACCOUNTS +} + +/// The amount transaction `k` of session `session` books. Every amount is +/// distinct, so a sum names the entries it counts. +fn amount(session: usize, k: usize) -> usize { + session * 100 + k + 1 +} + +/// Run session `session`'s transactions through `client`, one at a time. +/// Each returns only once it committed. +async fn book_entries(client: &tokio_postgres::Client, entries: &str, session: usize) { + for k in 0..ROUNDS { + let txn = format!( + "BEGIN; \ + INSERT INTO {entries} (id, account_id, amount) \ + VALUES ('s{session}e{k}', 'acc-{}', '{}'); \ + COMMIT", + account(session, k), + amount(session, k), + ); + run_retrying(client, &format!("session {session} txn {k}"), &txn).await; + } +} + +/// Move the leadership of each of `groups` to another of its replicas, +/// [`MOVES`] times, cycling through the replicas. +async fn move_leaders(cluster: &TestCluster, groups: &[u64]) { + for round in 0..MOVES { + for &group in groups { + let current = cluster + .wait_for_group_leader(group, GroupLeader::Any, LEADER_WAIT) + .await; + let others: Vec = group_members(&cluster.nodes[0], group) + .into_iter() + .filter(|member| *member != current) + .collect(); + assert!( + !others.is_empty(), + "group {group} has a replica besides its leader node {current}" + ); + let target = others[round % others.len()]; + cluster.transfer_leadership(group, target).await; + } + tokio::time::sleep(MOVE_PAUSE).await; + } +} + +/// A served number column as an integer. +fn number(text: &str, what: &str) -> usize { + text.parse::() + .map(|value| value as usize) + .unwrap_or_else(|e| panic!("{what} `{text}` is not a number: {e}")) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn calvin_txns_commit_once_while_data_group_leaders_move() { + let [accts, entries] = names_on_distinct_vshards(["split_accts", "split_entries"]); + let [accts_ddl, entries_ddl, sum] = sum_ddl(&entries, &accts); + let fx = Fixture::spawn(&[accts_ddl, entries_ddl]).await; + fx.cluster + .exec_ddl_on_any_leader(&sum) + .await + .unwrap_or_else(|e| panic!("{sum}: {e}")); + for collection in [&accts, &entries] { + fx.wait_group_mounted(collection).await; + } + for account in 0..ACCOUNTS { + run_retrying( + &fx.cluster.nodes[0].client, + "seed an account", + &format!( + "INSERT INTO {accts} (id, owner, balance) VALUES ('acc-{account}', 'seed', '0')" + ), + ) + .await; + } + fx.converge().await; + + let groups: Vec = [&accts, &entries] + .into_iter() + .map(|collection| group_of(&fx.cluster.nodes[0], collection)) + .collect::>() + .into_iter() + .collect(); + let sessions = [ + strict_session(&fx.cluster.nodes[1]).await, + strict_session(&fx.cluster.nodes[2]).await, + ]; + tokio::join!( + book_entries(&sessions[0], &entries, 0), + book_entries(&sessions[1], &entries, 1), + move_leaders(&fx.cluster, &groups), + ); + + fx.converge().await; + let collections = [accts.as_str(), entries.as_str()]; + wait_replicas_agree(&fx.cluster, &collections).await; + + let client = &fx.coordinator().client; + let mut stored = first_column(client, &format!("SELECT id FROM {entries}")).await; + stored.sort(); + let mut expected_ids: Vec = (0..SESSIONS) + .flat_map(|session| (0..ROUNDS).map(move |k| format!("s{session}e{k}"))) + .collect(); + expected_ids.sort(); + assert_eq!( + stored, expected_ids, + "each committed transaction's entry is stored once" + ); + + let mut committed: BTreeMap = BTreeMap::new(); + for session in 0..SESSIONS { + for k in 0..ROUNDS { + *committed.entry(account(session, k)).or_default() += amount(session, k); + } + } + for account in 0..ACCOUNTS { + let balances = first_column( + client, + &format!("SELECT balance FROM {accts} WHERE id = 'acc-{account}'"), + ) + .await; + let [balance] = balances.as_slice() else { + panic!("acc-{account} has one row, found balances {balances:?}"); + }; + let amounts = first_column( + client, + &format!("SELECT amount FROM {entries} WHERE account_id = 'acc-{account}'"), + ) + .await; + let summed: usize = amounts.iter().map(|a| number(a, "amount")).sum(); + let balance = number(balance, "balance"); + assert_eq!( + balance, summed, + "acc-{account}'s balance is the sum of its entries {amounts:?}" + ); + assert_eq!( + balance, + committed.get(&account).copied().unwrap_or_default(), + "acc-{account}'s balance counts each committed transaction once" + ); + } + + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs index 3136d886f..54900e568 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs @@ -23,12 +23,14 @@ mod calvin_multishard_dml_tag_fold; mod calvin_multishard_fixture; mod calvin_multishard_pk_read; mod calvin_multishard_txn_staging_cross_node; +mod calvin_old_term_redo_copy; mod calvin_ollp_cross_node; mod calvin_ollp_pk_delete; mod calvin_ollp_update; mod calvin_remote_participant_report; mod calvin_replica_content; mod calvin_replica_convergence; +mod calvin_split_leader; mod calvin_submit_routed_cross_node; mod calvin_superseded_collection_cluster; mod catalog_put_if_absent; diff --git a/nodedb-test-support/src/cluster_harness/cluster/leadership.rs b/nodedb-test-support/src/cluster_harness/cluster/leadership.rs new file mode 100644 index 000000000..4c5e5c562 --- /dev/null +++ b/nodedb-test-support/src/cluster_harness/cluster/leadership.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Data-group leadership moves and waits on a [`TestCluster`]. +//! +//! A move asks the group's leader to hand its leadership to a target, the +//! path the leader balance takes. The leader balance keeps running, so it +//! can move a group back to its preferred leader a few seconds later. + +use std::time::{Duration, Instant}; + +use super::TestCluster; +use crate::cluster_harness::wait::wait_for_report; + +/// Which node a wait expects to lead a group. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GroupLeader { + /// Any live node. + Any, + /// The node with this id. + Node(u64), +} + +/// How long one transfer request gets before the leader is asked again. +const TRANSFER_ROUND: Duration = Duration::from_secs(3); + +/// How long a leadership move gets in all. +const TRANSFER_DEADLINE: Duration = Duration::from_secs(30); + +/// Pause between two leadership reads. +const LEADER_POLL: Duration = Duration::from_millis(20); + +impl TestCluster { + /// The node that leads `group_id`: it leads in its own Raft state, and + /// every live node hosting the group names it leader. `None` while no + /// node leads or the nodes disagree. + pub fn group_leader(&self, group_id: u64) -> Option { + self.leader_report(group_id).ok() + } + + /// Wait until `leader` leads `group_id` as [`Self::group_leader`] reads + /// it, and return the leader's id. Panics at `timeout` with each node's + /// view of the group. + pub async fn wait_for_group_leader( + &self, + group_id: u64, + leader: GroupLeader, + timeout: Duration, + ) -> u64 { + let expected = match leader { + GroupLeader::Any => "any node".to_owned(), + GroupLeader::Node(node_id) => format!("node {node_id}"), + }; + let mut found = 0; + wait_for_report( + &format!("{expected} leads group {group_id}"), + timeout, + LEADER_POLL, + || { + let current = self.leader_report(group_id)?; + match leader { + GroupLeader::Node(node_id) if node_id != current => { + Err(format!("node {current} leads")) + } + GroupLeader::Any | GroupLeader::Node(_) => { + found = current; + Ok(()) + } + } + }, + ) + .await; + found + } + + /// Move the leadership of `group_id` to node `target`, and wait until + /// [`Self::group_leader`] names it. Each round asks the group's current + /// leader again: a transfer the leader refused, or one that lapsed, + /// starts nothing. Panics when the move does not finish within 30 s, + /// with the last refusal and each node's view. + pub async fn transfer_leadership(&self, group_id: u64, target: u64) { + let deadline = Instant::now() + TRANSFER_DEADLINE; + let mut last_refusal = String::from("no transfer was refused"); + loop { + if self.group_leader(group_id) == Some(target) { + return; + } + if Instant::now() >= deadline { + panic!( + "group {group_id} did not move to node {target} within \ + {TRANSFER_DEADLINE:?}; last refusal: {last_refusal}; views: {}", + self.leader_views(group_id) + ); + } + let leader = self.nodes.iter().find(|node| node.leads_group(group_id)); + if let Some(leader) = leader + && leader.node_id != target + && let Err(refusal) = leader.transfer_group_leadership(group_id, target) + { + last_refusal = refusal; + } + let round_end = Instant::now() + TRANSFER_ROUND; + while Instant::now() < round_end { + if self.group_leader(group_id) == Some(target) { + return; + } + tokio::time::sleep(LEADER_POLL).await; + } + } + } + + /// Wait until each data group is led by its preferred leader. The + /// leader balance then moves no group by itself, so the leaders stay + /// put until a test moves them. Panics at `timeout`. + pub async fn wait_for_preferred_leaders(&self, timeout: Duration) { + wait_for_report( + "every data group is led by its preferred leader", + timeout, + LEADER_POLL, + || { + let preferred = { + let routing = self.nodes[0] + .shared + .cluster_routing + .as_ref() + .ok_or_else(|| "node has no cluster routing".to_owned())? + .read() + .unwrap_or_else(|p| p.into_inner()); + nodedb_cluster::rebalancer::preferred_leaders(&routing) + }; + for (group_id, node_id) in preferred { + let current = self.leader_report(group_id)?; + if current != node_id { + return Err(format!( + "node {current} leads group {group_id}, node {node_id} is preferred" + )); + } + } + Ok(()) + }, + ) + .await; + } + + /// The agreed leader of `group_id`, or why there is none. + fn leader_report(&self, group_id: u64) -> Result { + let mut named = None; + for node in &self.nodes { + let Some(leader) = node + .all_group_leaders() + .into_iter() + .find(|(group, _)| *group == group_id) + .map(|(_, leader)| leader) + else { + continue; + }; + match named { + None => named = Some(leader), + Some(first) if first != leader => { + return Err(format!( + "the nodes disagree on group {group_id}'s leader: {}", + self.leader_views(group_id) + )); + } + Some(_) => {} + } + } + let leader = named + .filter(|leader| *leader != 0) + .ok_or_else(|| format!("no node leads group {group_id}"))?; + let leads = self + .nodes + .iter() + .any(|node| node.node_id == leader && node.leads_group(group_id)); + if leads { + Ok(leader) + } else { + Err(format!( + "node {leader} is named leader of group {group_id} but does not lead it" + )) + } + } + + /// Each live node's view of `group_id`'s leader, for a failure report. + fn leader_views(&self, group_id: u64) -> String { + self.nodes + .iter() + .map(|node| node.group_status_line(group_id)) + .collect::>() + .join("; ") + } +} diff --git a/nodedb-test-support/src/cluster_harness/cluster/mod.rs b/nodedb-test-support/src/cluster_harness/cluster/mod.rs index 63fd6d1e4..d0ed6e4ba 100644 --- a/nodedb-test-support/src/cluster_harness/cluster/mod.rs +++ b/nodedb-test-support/src/cluster_harness/cluster/mod.rs @@ -6,15 +6,18 @@ //! [`types`]; spawn-variant convenience wrappers in [`spawn_variants`]; //! the bringup body in [`bringup`]; the convergence barriers in [`ready`]; //! the in-place restart in [`restart`]; post-spawn membership + DDL helpers -//! in [`membership`]. +//! in [`membership`]; data-group leadership moves and waits in +//! [`leadership`]. mod bringup; +mod leadership; mod membership; mod ready; mod restart; mod spawn_variants; mod types; +pub use leadership::GroupLeader; pub use restart::{StoppedCluster, StoppedMember, StoppedNodeInfo}; pub use types::TestCluster; pub(crate) use types::{ClusterSpawnConfig, DEFAULT_NUM_GROUPS}; diff --git a/nodedb-test-support/src/cluster_harness/mod.rs b/nodedb-test-support/src/cluster_harness/mod.rs index ebabcb540..eb4f9f0e3 100644 --- a/nodedb-test-support/src/cluster_harness/mod.rs +++ b/nodedb-test-support/src/cluster_harness/mod.rs @@ -22,7 +22,7 @@ pub mod retriable; pub mod shared_steps; pub mod wait; -pub use cluster::{StoppedCluster, StoppedNodeInfo, TestCluster}; +pub use cluster::{GroupLeader, StoppedCluster, StoppedNodeInfo, TestCluster}; pub use node::TestClusterNode; pub use pitr::PitrStorage; pub use retriable::{is_no_serving_leader, read_once_a_leader_exists}; diff --git a/nodedb-test-support/src/cluster_harness/node/leadership.rs b/nodedb-test-support/src/cluster_harness/node/leadership.rs new file mode 100644 index 000000000..99b2a9c22 --- /dev/null +++ b/nodedb-test-support/src/cluster_harness/node/leadership.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Raft leadership reads and moves on one [`TestClusterNode`], through the +//! node's own Raft groups. + +use crate::cluster_harness::node::lifecycle::TestClusterNode; + +impl TestClusterNode { + /// Whether this node leads `group_id` in its own Raft state now. + pub fn leads_group(&self, group_id: u64) -> bool { + self.shared.multi_raft.get().is_some_and(|multi_raft| { + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .group_role_is_leader(group_id) + }) + } + + /// Ask this node, the leader of `group_id`, to hand the leadership to + /// node `target`. The move finishes once `target` wins the next term. + /// The error names the node, the group and the target. + pub fn transfer_group_leadership(&self, group_id: u64, target: u64) -> Result<(), String> { + let multi_raft = self + .shared + .multi_raft + .get() + .ok_or_else(|| format!("node {}: its Raft groups are not started", self.node_id))?; + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .transfer_leadership(group_id, target) + .map_err(|e| { + format!( + "node {}: transfer of group {group_id} to node {target}: {e}", + self.node_id + ) + }) + } +} diff --git a/nodedb-test-support/src/cluster_harness/node/mod.rs b/nodedb-test-support/src/cluster_harness/node/mod.rs index f2bbfa68b..19ac65c78 100644 --- a/nodedb-test-support/src/cluster_harness/node/mod.rs +++ b/nodedb-test-support/src/cluster_harness/node/mod.rs @@ -5,6 +5,7 @@ pub mod graph; pub mod inspect; +pub mod leadership; pub mod lifecycle; pub mod native_client; pub mod raw_pgwire; From c297404e02f97eae99696b638cce1dad072e48fc Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sat, 10 Oct 2026 14:04:59 +0800 Subject: [PATCH 13/19] feat(backup): order the consistent cut barrier with the Calvin marker The cut previously proposed the data-group barrier independently of the Calvin sequencer marker, so a transaction could land before the barrier in one group and after it in another. A backup or restore point could then hold a transaction on one group only. The marker now carries the barrier. Each group's leader proposes the barrier once its schedulers pass the marker, and holds the redo of every later transaction until it applies the barrier. Every slice of a transaction sequenced before the marker sits before the barrier in each group, and every slice of a later one sits after it. --- .../cases/calvin_backup_cut_order.rs | 307 ++++++++++++++++++ .../tests/common_suite/cases/mod.rs | 1 + nodedb-cluster/src/calvin/mod.rs | 5 +- nodedb-cluster/src/calvin/sequencer/entry.rs | 11 +- nodedb-cluster/src/calvin/sequencer/replay.rs | 12 +- .../calvin/sequencer/state_machine/apply.rs | 8 +- .../calvin/sequencer/state_machine/restore.rs | 29 +- .../src/calvin/types/cut_barrier.rs | 51 +++ nodedb-cluster/src/calvin/types/mod.rs | 2 + .../src/calvin/types/scheduler_input.rs | 10 +- .../src/physical_plan/meta_snapshot.rs | 2 +- nodedb/src/control/backup/capture.rs | 2 +- nodedb/src/control/backup/cut.rs | 254 ++++----------- .../src/control/backup/cut_capture/collect.rs | 9 +- .../control/backup/cut_capture/registry.rs | 5 +- nodedb/src/control/backup/cut_order/driver.rs | 227 +++++++++++++ nodedb/src/control/backup/cut_order/marker.rs | 106 ++++++ nodedb/src/control/backup/cut_order/mod.rs | 10 + .../control/backup/cut_order/ordered_cut.rs | 177 ++++++++++ .../src/control/backup/cut_order/registry.rs | 271 ++++++++++++++++ nodedb/src/control/backup/cut_order/wait.rs | 157 +++++++++ nodedb/src/control/backup/mod.rs | 1 + nodedb/src/control/backup/orchestrator.rs | 2 +- .../cluster/calvin/scheduler/cut_hold.rs | 129 ++++++++ .../scheduler/driver/core/cut_marker.rs | 269 ++++++++++++++- .../calvin/scheduler/driver/core/process.rs | 6 +- .../scheduler/driver/core/redo_propose.rs | 63 ++++ .../calvin/scheduler/driver/core/run_loop.rs | 15 + .../calvin/scheduler/driver/core/scheduler.rs | 4 + .../control/cluster/calvin/scheduler/mod.rs | 1 + .../cluster/calvin_snapshot/capture.rs | 1 + .../apply_loop/cut_barrier.rs | 202 ++++++++++++ .../distributed_applier/apply_loop/mod.rs | 3 + .../apply_loop/pipeline.rs | 2 + .../distributed_applier/apply_loop/start.rs | 153 +-------- .../src/control/exec_receiver/backup_cut.rs | 10 +- .../src/control/pitr/restore_point/record.rs | 15 +- .../server/exchange/all_cores/read_cut.rs | 1 + nodedb/src/control/state/calvin_cuts.rs | 34 ++ nodedb/src/control/state/calvin_local.rs | 4 + nodedb/src/diag/context/cut_barrier.rs | 48 +++ nodedb/src/diag/context/mod.rs | 2 + nodedb/src/diag/mod.rs | 4 +- nodedb/src/diag/recording/cut_barrier.rs | 32 ++ nodedb/src/diag/recording/mod.rs | 2 + 45 files changed, 2285 insertions(+), 374 deletions(-) create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_order.rs create mode 100644 nodedb-cluster/src/calvin/types/cut_barrier.rs create mode 100644 nodedb/src/control/backup/cut_order/driver.rs create mode 100644 nodedb/src/control/backup/cut_order/marker.rs create mode 100644 nodedb/src/control/backup/cut_order/mod.rs create mode 100644 nodedb/src/control/backup/cut_order/ordered_cut.rs create mode 100644 nodedb/src/control/backup/cut_order/registry.rs create mode 100644 nodedb/src/control/backup/cut_order/wait.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/cut_hold.rs create mode 100644 nodedb/src/control/distributed_applier/apply_loop/cut_barrier.rs create mode 100644 nodedb/src/diag/context/cut_barrier.rs create mode 100644 nodedb/src/diag/recording/cut_barrier.rs diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_order.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_order.rs new file mode 100644 index 000000000..e3af27706 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_order.rs @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A database backup taken while cross-shard Calvin transactions commit +//! restores whole transactions: no transaction is in the backup on one data +//! group only. +//! +//! 1. A 3-node cluster with one backup root holds `accts`, whose `balance` is +//! the materialized sum of `entries.amount`, and `entries`, in distinct +//! data groups. Every account starts at 100. +//! 2. The fail gate `calvin::before_redo_propose::` holds the +//! entries slice of every transaction after its resolve, on the entries +//! group's leader. Transaction `held` adds 25 to `acc-0`: its balance +//! slice installs, its entries slice waits. +//! 3. Writers commit more cross-shard transactions on the other accounts. +//! `BACKUP DATABASE default` starts meanwhile. Its cut's marker follows +//! `held` in the sequencer log, so the entries group's barrier must follow +//! `held`'s entries redo, which is still unproposed. The backup stays +//! unfinished. +//! 4. The test releases the gate. The held redo installs, the barriers +//! follow, and the writers keep committing while the backup finishes. +//! 5. A fresh cluster restores the backup. `held`'s entry is in it, and every +//! account's balance is 100 plus the amounts of its restored entries. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use nodedb_types::fail_point::{FailAction, FailGuard}; + +use super::calvin_multishard_fixture::Fixture; +use super::calvin_replica_content::{run_retrying, strict_session, sum_ddl, vshard_of}; +use crate::common::cluster_harness::shared_steps::{db_detail, name_where}; +use crate::common::cluster_harness::{TestCluster, wait_for, wait_for_async}; + +/// Every account, each starting at [`BASE`]. +const ACCOUNTS: [&str; 4] = ["acc-0", "acc-1", "acc-2", "acc-3"]; + +/// The balance every account starts at. +const BASE: i64 = 100; + +/// The amount the held transaction adds to `acc-0`. +const HELD_AMOUNT: i64 = 25; + +/// How long each step gets. +const STEP_DEADLINE: Duration = Duration::from_secs(30); + +/// Pause between two polls of a step. +const STEP_POLL: Duration = Duration::from_millis(20); + +/// How long the backup must stay unfinished while the held redo waits. +const PARKED_FOR: Duration = Duration::from_millis(1500); + +/// Most transactions one writer commits. +const MAX_WRITES: u32 = 40; + +/// The backup object, under the shared backup root. +const OBJECT: &str = "calvin_cut/default.ndbb"; + +/// Every row `sql` returns through `client`, each column as text. +async fn rows(client: &tokio_postgres::Client, sql: &str) -> Vec> { + client + .simple_query(sql) + .await + .unwrap_or_else(|e| panic!("`{sql}`: {}", db_detail(&e))) + .into_iter() + .filter_map(|msg| match msg { + tokio_postgres::SimpleQueryMessage::Row(row) => Some( + (0..row.len()) + .map(|idx| row.get(idx).unwrap_or_default().to_owned()) + .collect(), + ), + _ => None, + }) + .collect() +} + +/// Commit cross-shard transactions that add to `account`, one at a time, +/// until `stop` or [`MAX_WRITES`]. Returns how many committed. +async fn write_until( + client: tokio_postgres::Client, + entries: String, + account: &'static str, + stop: Arc, +) -> u32 { + let mut written = 0; + while !stop.load(Ordering::Relaxed) && written < MAX_WRITES { + let sql = format!( + "BEGIN; \ + INSERT INTO {entries} (id, account_id, amount) \ + VALUES ('{account}-w{written}', '{account}', '{}'); \ + COMMIT", + written % 7 + 1 + ); + run_retrying(&client, "a concurrent Calvin commit", &sql).await; + written += 1; + } + written +} + +/// Run `sql` on node 0 of `cluster`. Panics with the server's error. +async fn exec(cluster: &TestCluster, sql: &str) { + cluster.nodes[0] + .client + .simple_query(sql) + .await + .unwrap_or_else(|e| panic!("`{sql}`: {}", db_detail(&e))); +} + +/// Read `accts` and `entries` on node 1 of `cluster`, and check that each +/// account's balance is [`BASE`] plus the amounts of its entries, and that +/// the held transaction's entry is there. +async fn assert_whole_transactions(cluster: &TestCluster, accts: &str, entries: &str) { + let client = &cluster.nodes[1].client; + let balances = rows(client, &format!("SELECT id, balance FROM {accts}")).await; + let restored = rows( + client, + &format!("SELECT id, account_id, amount FROM {entries}"), + ) + .await; + assert!( + restored.iter().any(|row| row[0] == "held"), + "the held transaction was sequenced before the backup's cut, so its entry is in \ + the backup; restored entries: {restored:?}" + ); + assert_eq!( + balances.len(), + ACCOUNTS.len(), + "every account is restored: {balances:?}" + ); + for row in &balances { + let account = &row[0]; + let balance: i64 = row[1] + .parse() + .unwrap_or_else(|e| panic!("{account}'s balance '{}': {e}", row[1])); + let added: i64 = restored + .iter() + .filter(|entry| entry[1] == *account) + .map(|entry| { + entry[2] + .parse::() + .unwrap_or_else(|e| panic!("entry {}'s amount '{}': {e}", entry[0], entry[2])) + }) + .sum(); + assert_eq!( + balance, + BASE + added, + "{account}: a transaction is in the backup on one data group only. Balance \ + {balance}, base {BASE} plus restored entries {added}" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 8)] +async fn a_backup_during_calvin_commits_restores_whole_transactions() { + let root_dir = tempfile::tempdir().expect("backup root"); + let root = root_dir + .path() + .canonicalize() + .expect("canonical backup root"); + let uri = format!("file://{}/{OBJECT}", root.display()); + let source = TestCluster::spawn_three_with_backup_root(root.clone()) + .await + .expect("source cluster"); + let fx = Fixture::from_cluster(source).await; + let probe = &fx.cluster.nodes[0]; + let entries = name_where("cut_order_entries", |name| { + probe.group_id_for_collection(name).is_some() + }); + let group = probe + .group_id_for_collection(&entries) + .expect("the entries vShard maps to a group"); + let accts = name_where("cut_order_accts", |name| { + vshard_of(name) != vshard_of(&entries) + && probe + .group_id_for_collection(name) + .is_some_and(|other| other != group) + }); + let [accts_ddl, entries_ddl, sum] = sum_ddl(&entries, &accts); + fx.create(&[accts_ddl, entries_ddl]).await; + fx.cluster + .exec_ddl_on_any_leader(&sum) + .await + .unwrap_or_else(|e| panic!("{sum}: {e}")); + for collection in [&accts, &entries] { + fx.wait_group_mounted(collection).await; + } + for account in ACCOUNTS { + run_retrying( + &fx.cluster.nodes[0].client, + "seed an account", + &format!( + "INSERT INTO {accts} (id, owner, balance) VALUES ('{account}', 'seed', '{BASE}')" + ), + ) + .await; + } + fx.converge().await; + + let gate_dir = tempfile::tempdir().expect("gate directory"); + let release = gate_dir.path().join("propose"); + let parked = gate_dir.path().join("propose.parked"); + let gate = FailGuard::install( + &format!("calvin::before_redo_propose::{entries}"), + FailAction::WaitForFile(release.clone()), + ); + + // The held transaction: its balance slice installs, its entries slice + // waits at the gate. + let held_client = strict_session(fx.coordinator()).await; + let held_sql = format!( + "BEGIN; \ + INSERT INTO {entries} (id, account_id, amount) VALUES ('held', 'acc-0', '{HELD_AMOUNT}'); \ + COMMIT" + ); + let held = tokio::spawn(async move { + held_client + .simple_query(&held_sql) + .await + .map(|_| ()) + .map_err(|e| db_detail(&e)) + }); + wait_for( + "the entries leader holds the held slice's redo", + STEP_DEADLINE, + STEP_POLL, + || parked.exists(), + ) + .await; + let coordinator = fx.coordinator(); + let accts_name = accts.as_str(); + let held_balance = (BASE + HELD_AMOUNT).to_string(); + wait_for_async( + "the held transaction's balance slice installs", + STEP_DEADLINE, + STEP_POLL, + || { + let held_balance = held_balance.clone(); + async move { + rows( + &coordinator.client, + &format!("SELECT balance FROM {accts_name} WHERE id = 'acc-0'"), + ) + .await + == vec![vec![held_balance]] + } + }, + ) + .await; + + // Writers on the other accounts, before and after the backup's cut. + let stop = Arc::new(AtomicBool::new(false)); + let mut writers = Vec::new(); + for (idx, account) in ACCOUNTS.into_iter().skip(1).enumerate() { + let node = &fx.cluster.nodes[idx % fx.cluster.nodes.len()]; + let client = strict_session(node).await; + writers.push(tokio::spawn(write_until( + client, + entries.clone(), + account, + Arc::clone(&stop), + ))); + } + + let backup_client = strict_session(&fx.cluster.nodes[0]).await; + let backup_sql = format!("BACKUP DATABASE default TO '{uri}'"); + let backup = tokio::spawn(async move { + backup_client + .simple_query(&backup_sql) + .await + .map(|_| ()) + .map_err(|e| db_detail(&e)) + }); + tokio::time::sleep(PARKED_FOR).await; + assert!( + !backup.is_finished(), + "the backup finished while a transaction sequenced before its cut was not installed" + ); + + std::fs::write(&release, b"release").expect("release the held redo"); + held.await + .expect("held transaction task") + .unwrap_or_else(|e| panic!("the held transaction: {e}")); + backup + .await + .expect("backup task") + .unwrap_or_else(|e| panic!("BACKUP DATABASE: {e}")); + stop.store(true, Ordering::Relaxed); + for writer in writers { + let written = writer.await.expect("writer task"); + assert!(written > 0, "every writer committed while the backup ran"); + } + drop(gate); + assert!(root.join(OBJECT).exists(), "the backup wrote its object"); + fx.cluster.shutdown().await; + + let target = TestCluster::spawn_three_with_backup_root(root.clone()) + .await + .expect("target cluster"); + exec(&target, &format!("RESTORE DATABASE default FROM '{uri}'")).await; + target.wait_for_full_apply_convergence(STEP_DEADLINE).await; + assert_whole_transactions(&target, &accts, &entries).await; + target.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs index 54900e568..fdd6b81f5 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs @@ -11,6 +11,7 @@ mod array_vshard_migration; mod assign_surrogate_cross_node; mod backup_schedule_failover; mod bitemporal_array_cluster; +mod calvin_backup_cut_order; mod calvin_cdc_net_kinds; mod calvin_cluster_pgwire_e2e; mod calvin_duplicate_redo_copy; diff --git a/nodedb-cluster/src/calvin/mod.rs b/nodedb-cluster/src/calvin/mod.rs index f46bffc0a..c66504177 100644 --- a/nodedb-cluster/src/calvin/mod.rs +++ b/nodedb-cluster/src/calvin/mod.rs @@ -26,4 +26,7 @@ pub use sequencer::{ SequencerStateMachine, UnrecoverableEpochHook, new_inbox, new_reservation_inbox, validate_batch, }; -pub use types::{EngineKeySet, EpochBatch, ReadWriteSet, SequencedTxn, SortedVec, TxClass}; +pub use types::{ + CutBarrierWire, CutCaptureWire, EngineKeySet, EpochBatch, ReadWriteSet, SequencedTxn, + SortedVec, TxClass, +}; diff --git a/nodedb-cluster/src/calvin/sequencer/entry.rs b/nodedb-cluster/src/calvin/sequencer/entry.rs index a4e59274a..9774f2232 100644 --- a/nodedb-cluster/src/calvin/sequencer/entry.rs +++ b/nodedb-cluster/src/calvin/sequencer/entry.rs @@ -138,7 +138,16 @@ pub enum SequencerEntry { /// `restore_point` names the cluster restore point the cut takes, `0` for /// a backup's cut. Every replica records the sequencer's place at the /// point when it applies the marker. - CutMarker { hlc: u64, restore_point: u64 }, + /// + /// `barrier` is set when the cut places a barrier in every data group. + /// Each group's leader then proposes the barrier once its schedulers + /// passed the marker, and holds the redo of every later transaction + /// until it applied the barrier. `None` for a cut with no barrier. + CutMarker { + hlc: u64, + restore_point: u64, + barrier: Option, + }, /// The first entry of a sequencer log a cluster restore rebuilt. It sets /// the next epoch the sequencer proposes to `next_epoch`, the epoch that /// followed the restore point, so no restored epoch is minted again. It diff --git a/nodedb-cluster/src/calvin/sequencer/replay.rs b/nodedb-cluster/src/calvin/sequencer/replay.rs index d5cae84de..8a62a6a03 100644 --- a/nodedb-cluster/src/calvin/sequencer/replay.rs +++ b/nodedb-cluster/src/calvin/sequencer/replay.rs @@ -168,8 +168,16 @@ impl SequencerStateMachine { } // A cut marker reaches every vShard, exactly as the live // `CutMarker` arm fans it out. - SequencerEntry::CutMarker { hlc, .. } => { - result.push(SchedulerInput::CutMarker { hlc }); + SequencerEntry::CutMarker { + hlc, + restore_point, + barrier, + } => { + result.push(SchedulerInput::CutMarker { + hlc, + restore_point, + barrier: barrier.map(std::sync::Arc::new), + }); } // A part reaches the vShards it targets, exactly as the live // `TxnPart` arm fans it out. The scheduler ignores a part of a diff --git a/nodedb-cluster/src/calvin/sequencer/state_machine/apply.rs b/nodedb-cluster/src/calvin/sequencer/state_machine/apply.rs index 64c413186..0015a141c 100644 --- a/nodedb-cluster/src/calvin/sequencer/state_machine/apply.rs +++ b/nodedb-cluster/src/calvin/sequencer/state_machine/apply.rs @@ -161,9 +161,11 @@ impl SequencerStateMachine { next_epoch, epoch_system_ms, } => self.apply_epoch_floor(next_epoch, epoch_system_ms), - SequencerEntry::CutMarker { hlc, restore_point } => { - self.apply_cut_marker(index, hlc, restore_point) - } + SequencerEntry::CutMarker { + hlc, + restore_point, + barrier, + } => self.apply_cut_marker(index, hlc, restore_point, barrier), SequencerEntry::TxnPart { epoch, position, diff --git a/nodedb-cluster/src/calvin/sequencer/state_machine/restore.rs b/nodedb-cluster/src/calvin/sequencer/state_machine/restore.rs index f66828787..40591c3e5 100644 --- a/nodedb-cluster/src/calvin/sequencer/state_machine/restore.rs +++ b/nodedb-cluster/src/calvin/sequencer/state_machine/restore.rs @@ -4,9 +4,11 @@ //! backup or restore point takes, and the epoch floor a restored log opens //! with. +use std::sync::Arc; + use tracing::warn; -use crate::calvin::types::SchedulerInput; +use crate::calvin::types::{CutBarrierWire, SchedulerInput}; use super::core::{Delivery, NOT_YET_APPLIED, SequencerRestorePoint, SequencerStateMachine}; @@ -29,7 +31,14 @@ impl SequencerStateMachine { /// out to every vShard scheduler this node hosts. Same delivery as /// `ReserveRead`: a marker for an armed vShard, or one dropped at a full /// channel, is replayed by the scheduler's catch-up drain in log order. - pub(super) fn apply_cut_marker(&mut self, index: u64, hlc: u64, restore_point: u64) { + /// Every scheduler shares one copy of the marker's `barrier`. + pub(super) fn apply_cut_marker( + &mut self, + index: u64, + hlc: u64, + restore_point: u64, + barrier: Option, + ) { if restore_point != 0 && let Some(hook) = &self.restore_point_hook { @@ -46,9 +55,15 @@ impl SequencerStateMachine { if let Some(hook) = &self.cut_instant_hook { hook(hlc, index, self.last_epoch_system_ms); } + let barrier = barrier.map(Arc::new); let vshards: Vec = self.vshard_senders.keys().copied().collect(); for vshard in vshards { - match self.deliver(index, vshard, SchedulerInput::CutMarker { hlc }) { + let marker = SchedulerInput::CutMarker { + hlc, + restore_point, + barrier: barrier.clone(), + }; + match self.deliver(index, vshard, marker) { Delivery::NotHosted | Delivery::Sent | Delivery::Deferred => {} Delivery::DroppedFull => warn!( vshard, @@ -128,6 +143,7 @@ mod tests { &encode(&SequencerEntry::CutMarker { hlc: 900, restore_point: 0, + barrier: None, }), ); sm.apply( @@ -135,6 +151,7 @@ mod tests { &encode(&SequencerEntry::CutMarker { hlc: 1_000, restore_point: 77, + barrier: Some(CutBarrierWire { capture: None }), }), ); @@ -149,10 +166,11 @@ mod tests { }], "only a restore point's marker is reported" ); - for hlc in [900, 1_000] { + for (hlc, point, ordered) in [(900, 0, false), (1_000, 77, true)] { assert!(matches!( rx.try_recv(), - Ok(SchedulerInput::CutMarker { hlc: got }) if got == hlc + Ok(SchedulerInput::CutMarker { hlc: got, restore_point, barrier }) + if got == hlc && restore_point == point && barrier.is_some() == ordered )); } } @@ -172,6 +190,7 @@ mod tests { encode(&SequencerEntry::CutMarker { hlc, restore_point: 0, + barrier: None, }) }; sm.apply(1, &marker(10)); diff --git a/nodedb-cluster/src/calvin/types/cut_barrier.rs b/nodedb-cluster/src/calvin/types/cut_barrier.rs new file mode 100644 index 000000000..50f208e31 --- /dev/null +++ b/nodedb-cluster/src/calvin/types/cut_barrier.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Plain-data wire twins of the data-group barrier a backup's cut orders. +//! +//! A cut marker that carries a [`CutBarrierWire`] asks every data group for +//! one barrier at the marker's watermark. The leader of each group proposes +//! the barrier once every scheduler of the group's vShards on that node +//! passed the marker. Until it applied the barrier, the leader holds the +//! redo of every transaction sequenced after the marker. So every slice of a +//! transaction before the marker sits before the barrier in its group, and +//! every slice of a later one sits after it. +//! +//! The sequencer crate cannot depend on the host's capture request, so +//! [`CutCaptureWire`] mirrors its fields. + +use serde::{Deserialize, Serialize}; + +/// Wire twin of a database backup's capture request. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CutCaptureWire { + /// Unique per backup. It keys every parked capture. + pub request_id: u64, + pub database_id: u64, + pub tenants: Vec, +} + +/// The barrier a cut marker orders into every data group. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct CutBarrierWire { + /// The capture each group's leader takes when it applies the barrier, + /// `None` for a cut that captures nothing. + pub capture: Option, +} diff --git a/nodedb-cluster/src/calvin/types/mod.rs b/nodedb-cluster/src/calvin/types/mod.rs index 0828bd18d..18dcfde82 100644 --- a/nodedb-cluster/src/calvin/types/mod.rs +++ b/nodedb-cluster/src/calvin/types/mod.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 +pub mod cut_barrier; pub mod lock_wire; pub mod multi_part; pub mod primitives; @@ -8,6 +9,7 @@ pub mod scheduler_input; pub mod sequencer; pub mod transaction; +pub use cut_barrier::{CutBarrierWire, CutCaptureWire}; pub use lock_wire::{LockKeyWire, ReleaseReason, TxnIdWire}; pub use multi_part::{ MultiPartPlans, PartStreamId, PlanPart, StreamedPart, TaskChunk, VShardParts, diff --git a/nodedb-cluster/src/calvin/types/scheduler_input.rs b/nodedb-cluster/src/calvin/types/scheduler_input.rs index 23724d495..2704a5318 100644 --- a/nodedb-cluster/src/calvin/types/scheduler_input.rs +++ b/nodedb-cluster/src/calvin/types/scheduler_input.rs @@ -36,7 +36,15 @@ pub enum SchedulerInput { /// A backup's consistent-cut marker carrying its watermark `hlc`. Every /// transaction delivered before it must finish before the scheduler /// reports it; every transaction delivered after it commits above `hlc`. - CutMarker { hlc: u64 }, + /// `restore_point` names the cluster restore point the cut takes, `0` + /// for none. `barrier` is set when the cut places a barrier in every data + /// group: the leader holds the redo of every later transaction until it + /// applied its group's barrier. Shared by every vShard of this node. + CutMarker { + hlc: u64, + restore_point: u64, + barrier: Option>, + }, /// Part `index` of the plans of the multi-part transaction `txn`, whose /// first task is the transaction's task `first_task`. The bytes are /// shared by every scheduler of this node the part targets. `chunk` is diff --git a/nodedb-physical/src/physical_plan/meta_snapshot.rs b/nodedb-physical/src/physical_plan/meta_snapshot.rs index 0af8302af..d83aedf0f 100644 --- a/nodedb-physical/src/physical_plan/meta_snapshot.rs +++ b/nodedb-physical/src/physical_plan/meta_snapshot.rs @@ -29,7 +29,7 @@ pub struct SnapshotClearTarget { /// A database backup's request to capture its tenants at its cut barrier. /// -/// The barrier the backup proposes into each data group carries it. The +/// The barrier each data group's leader places for the backup carries it. The /// group's leader, when it applies the first barrier of `request_id`, /// snapshots every tenant of `tenants` in `database_id` before it applies the /// next entry, and parks the capture for the backup to collect. diff --git a/nodedb/src/control/backup/capture.rs b/nodedb/src/control/backup/capture.rs index d8d58ed4e..d2b0f26ed 100644 --- a/nodedb/src/control/backup/capture.rs +++ b/nodedb/src/control/backup/capture.rs @@ -39,7 +39,7 @@ pub async fn capture_collections( arrays: bool, ) -> Result { let assignment = source_assignment(state)?; - let watermark = super::cut::consistent_cut(state, tenant_id).await?; + let watermark = super::cut::consistent_cut(state).await?; let per_node = join_all( assignment diff --git a/nodedb/src/control/backup/cut.rs b/nodedb/src/control/backup/cut.rs index ed70e709a..5ea8eed68 100644 --- a/nodedb/src/control/backup/cut.rs +++ b/nodedb/src/control/backup/cut.rs @@ -12,63 +12,68 @@ //! //! Three waits make the cut: //! -//! - **Raft data groups.** A replicated write carries its proposer's commit -//! stamp and applies in log order. The cut proposes a -//! [`ReplicatedWrite::CutBarrier`] carrying `W` into every data group this -//! node hosts and waits for this node's apply of it. Every entry before the -//! barrier applied first. Every entry after it records a commit HLC above -//! `W`, however early its proposer stamped it. A database backup's barrier -//! also captures its tenants at that log position (see -//! [`super::cut_capture`]). //! - **Calvin transactions.** A Calvin transaction commits at its place in //! the sequencer log. The cut proposes a `CutMarker` carrying `W` into the //! sequencer log and waits until every Calvin scheduler on this node passed //! it: every transaction delivered before the marker installed or dropped. //! Every transaction delivered after it records a commit HLC above `W`. +//! - **Raft data groups.** A replicated write carries its proposer's commit +//! stamp and applies in log order. The cut places a +//! [`ReplicatedWrite::CutBarrier`] carrying `W` in every data group and +//! waits for this node's apply of the barrier of each group it hosts. +//! Every entry before the barrier applied first. Every entry after it +//! records a commit HLC above `W`, however early its proposer stamped it. A +//! database backup's barrier also captures its tenants at that log +//! position (see [`super::cut_capture`]). //! - **Local write windows.** A write the funnel appends here stamps itself //! after its record is minted inside an outcome-floor window. The cut reads //! the highest LSN any window minted, after it picks `W`, and waits for the //! outcome floor to reach it. A record minted later stamps itself above `W`. //! +//! A Calvin slice installs from a stamped redo entry its group's leader +//! proposes into the data-group log. The barrier is ordered with the marker +//! (see [`super::cut_order`]): the marker carries the barrier, and each +//! group's leader proposes it once its schedulers passed the marker, while +//! it holds the redo of every later transaction until it applied the +//! barrier. In every group, each slice of a transaction sequenced before the +//! marker sits before the barrier, and each slice of a later one after it, +//! so no transaction is in a capture or a restore point on one group only. +//! //! Two stamps can share a wall time. Once it picks `W`, the cut moves this //! node's clock past `W`, so every later stamp here reads above it. //! //! The waits bind this node's replicas. A remote source node takes the same //! cut at `W` itself before it snapshots: the snapshot plan sent to it carries //! `W`, and its Control Plane runs [`cut_at`] first. +//! +//! [`ReplicatedWrite::CutBarrier`]: crate::control::wal_replication::ReplicatedWrite::CutBarrier -use std::collections::BTreeMap; use std::time::Duration; -use nodedb_cluster::METADATA_GROUP_ID; -use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; -use nodedb_cluster::calvin::SequencerEntry; use nodedb_physical::physical_plan::CutCaptureRequest; use nodedb_types::Hlc; use crate::Error; -use crate::control::security::auth_fence::cluster::{group_of_vshard, hosts_group, routed_groups}; use crate::control::state::SharedState; -use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite, propose_replicated_entry}; -use crate::types::{DatabaseId, VShardId}; + +use super::cut_order::OrderedCut; +use super::cut_order::driver::{ensure_driver, hosted_data_groups}; +use super::cut_order::marker::{Marker, place_marker}; +use super::cut_order::wait::await_group_barriers; /// Pick the envelope watermark and wait until every write committed below it /// has a final outcome on this node. Returns the watermark. -pub(super) async fn consistent_cut(state: &SharedState, tenant_id: u64) -> Result { +pub(super) async fn consistent_cut(state: &SharedState) -> Result { let watermark = state.hlc_clock.now().wall_ns; - cut_at(state, tenant_id, watermark).await?; + cut_at(state, watermark).await?; Ok(watermark) } /// Take the consistent cut at `watermark` on this node: wait until every /// write committed below it has a final outcome here. A remote source node /// runs it for the watermark the backup's coordinator picked. -pub(crate) async fn cut_at( - state: &SharedState, - tenant_id: u64, - watermark: u64, -) -> Result<(), Error> { - cut_at_point(state, tenant_id, watermark, 0).await +pub(crate) async fn cut_at(state: &SharedState, watermark: u64) -> Result<(), Error> { + cut_at_point(state, watermark, 0).await } /// [`cut_at`] for the cluster restore point `restore_point`, `0` for a @@ -76,11 +81,18 @@ pub(crate) async fn cut_at( /// every replica records its group's place at the point. pub(crate) async fn cut_at_point( state: &SharedState, - tenant_id: u64, watermark: u64, restore_point: u64, ) -> Result<(), Error> { - cut_barriers(state, tenant_id, watermark, restore_point, None).await + cut_barriers( + state, + OrderedCut { + hlc: watermark, + restore_point, + capture: None, + }, + ) + .await } /// [`cut_at`] whose barriers carry a database backup's capture `request`. @@ -88,35 +100,46 @@ pub(crate) async fn cut_at_point( /// when it applies the request's first barrier in that group. pub(crate) async fn cut_with_capture( state: &SharedState, - tenant_id: u64, watermark: u64, request: &CutCaptureRequest, ) -> Result<(), Error> { - cut_barriers(state, tenant_id, watermark, 0, Some(request)).await + cut_barriers( + state, + OrderedCut { + hlc: watermark, + restore_point: 0, + capture: Some(request.clone()), + }, + ) + .await } -async fn cut_barriers( - state: &SharedState, - tenant_id: u64, - watermark: u64, - restore_point: u64, - capture: Option<&CutCaptureRequest>, -) -> Result<(), Error> { +async fn cut_barriers(state: &SharedState, cut: OrderedCut) -> Result<(), Error> { state .hlc_clock - .update(Hlc::new(watermark.saturating_add(1), 0)); + .update(Hlc::new(cut.hlc.saturating_add(1), 0)); let timeout = Duration::from_secs(state.tuning.network.default_deadline_secs); let deadline = tokio::time::Instant::now() + timeout; // Read after the watermark: a record minted after this read stamps its // write above the watermark. let target = state.outcome_floor.max_noted(); - let (groups, calvin) = tokio::join!( - cut_data_groups(state, tenant_id, watermark, restore_point, capture), - cut_calvin_point(state, watermark, restore_point, deadline), + let groups = hosted_data_groups(state); + if !groups.is_empty() { + // This node places the barrier of every group it leads. + ensure_driver(&state.self_arc()?, &cut); + } + let marker = Marker { + hlc: cut.hlc, + restore_point: cut.restore_point, + barrier: Some(&cut), + }; + let (barriers, calvin) = tokio::join!( + await_group_barriers(state, &cut, &groups, deadline), + place_marker(state, marker, &groups, deadline), ); - groups?; calvin?; + barriers?; if !state.outcome_floor.await_floor(target, deadline).await { return Err(Error::Internal { @@ -134,156 +157,17 @@ async fn cut_barriers( Ok(()) } -/// Propose a cut barrier carrying `watermark` into every data group this node -/// hosts, and wait for this node's apply of each. -async fn cut_data_groups( - state: &SharedState, - tenant_id: u64, - watermark: u64, - restore_point: u64, - capture: Option<&CutCaptureRequest>, -) -> Result<(), Error> { - let proposer = state.async_raft_proposer()?; - let deadline = crate::control::wal_replication::statement_propose_deadline(state); - let barriers = futures::future::join_all(barrier_vshards(state).into_iter().map( - |(group_id, vshard_id)| { - // The barrier orders every entry of its group, whatever database - // the entry writes, so one barrier per group cuts every database. - // The entry's database id only frames it. - let entry = ReplicatedEntry::new( - tenant_id, - DatabaseId::DEFAULT.as_u64(), - vshard_id, - ReplicatedWrite::CutBarrier { - hlc: watermark, - restore_point, - capture: capture.cloned(), - }, - ); - async move { - propose_replicated_entry(state, proposer, entry, deadline) - .await - .map_err(|error| (group_id, error)) - } - }, - )) - .await; - for barrier in barriers { - if let Err((group_id, error)) = barrier { - // A group this node left while the barrier waited holds no - // replica here to cut: the source node that snapshots it takes - // its own cut. - if !hosts_group(state, group_id) { - tracing::info!( - group_id, - %error, - "backup: this node left the group before its cut barrier applied here; \ - the group needs no cut on this node" - ); - continue; - } - return Err(Error::Internal { - detail: format!( - "backup: the consistent-cut barrier of raft group {group_id} did not \ - apply on this node: {error}. Retry the backup" - ), - }); - } - } - Ok(()) -} - -/// How long the cut waits for its Calvin marker before it proposes the -/// marker again. A leader change can drop a proposed marker; a second copy is -/// harmless, since a scheduler passes each marker once its earlier -/// transactions finished. -const CUT_MARKER_RETRY: Duration = Duration::from_secs(1); - -/// Propose a Calvin cut marker carrying `watermark`, and wait until every -/// Calvin scheduler on this node passed it, or `deadline`. +/// Propose a Calvin cut marker carrying `watermark` and no barrier, and wait +/// until every Calvin scheduler on this node passed it, or `deadline`. pub(crate) async fn cut_calvin( state: &SharedState, watermark: u64, deadline: tokio::time::Instant, ) -> Result<(), Error> { - cut_calvin_point(state, watermark, 0, deadline).await -} - -/// [`cut_calvin`] with a marker that carries the cluster restore point -/// `restore_point`, `0` for none. -async fn cut_calvin_point( - state: &SharedState, - watermark: u64, - restore_point: u64, - deadline: tokio::time::Instant, -) -> Result<(), Error> { - let cuts = &state.calvin.cuts; - if cuts.is_empty() { - // No Calvin scheduler runs here: this node applies no Calvin write. - return Ok(()); - } - let proposer = state - .calvin - .sequencer_proposer - .get() - .ok_or_else(|| Error::Internal { - detail: "backup: Calvin schedulers run on this node, but no sequencer proposer \ - is set, so the consistent cut cannot place its marker. Retry the backup \ - once the cluster finished starting" - .into(), - })?; - let marker = zerompk::to_msgpack_vec(&SequencerEntry::CutMarker { + let marker = Marker { hlc: watermark, - restore_point, - }) - .map_err(|error| Error::Internal { - detail: format!("backup: encode the Calvin cut marker: {error}"), - })?; - let mut last_refusal = None; - loop { - if let Err(error) = proposer.propose(marker.clone()) { - last_refusal = Some(error.to_string()); - } - let attempt_deadline = deadline.min(tokio::time::Instant::now() + CUT_MARKER_RETRY); - let lagging = cuts.await_passed(watermark, attempt_deadline).await; - if lagging.is_empty() { - return Ok(()); - } - if tokio::time::Instant::now() >= deadline { - return Err(Error::Internal { - detail: format!( - "backup: the Calvin schedulers of vShards {lagging:?} did not pass the \ - consistent-cut marker in time, so Calvin transactions sequenced before \ - the backup may not be installed (last marker refusal: {}). Retry the \ - backup", - last_refusal.as_deref().unwrap_or("none") - ), - }); - } - } -} - -/// One vShard per data group this node hosts: the barrier of a group is -/// proposed through any vShard the group homes. -fn barrier_vshards(state: &SharedState) -> BTreeMap { - let hosted: Vec = routed_groups(state) - .into_iter() - .filter(|group_id| { - *group_id != METADATA_GROUP_ID - && *group_id != SEQUENCER_GROUP_ID - && hosts_group(state, *group_id) - }) - .collect(); - let mut vshards = BTreeMap::new(); - for vshard_id in 0..VShardId::COUNT { - if vshards.len() == hosted.len() { - break; - } - if let Ok(group_id) = group_of_vshard(state, vshard_id) - && hosted.contains(&group_id) - { - vshards.entry(group_id).or_insert(vshard_id); - } - } - vshards + restore_point: 0, + barrier: None, + }; + place_marker(state, marker, &[], deadline).await } diff --git a/nodedb/src/control/backup/cut_capture/collect.rs b/nodedb/src/control/backup/cut_capture/collect.rs index f37668f4d..fca3dcb87 100644 --- a/nodedb/src/control/backup/cut_capture/collect.rs +++ b/nodedb/src/control/backup/cut_capture/collect.rs @@ -4,7 +4,7 @@ //! //! The coordinator picks the cut watermark and a request id, then takes the //! cut with capturing barriers on its own Raft data groups, a one-node -//! cluster's included. Every other source node receives the request, takes +//! cluster's included. Each group's leader places the group's one barrier. Every other source node receives the request, takes //! the same cut on its groups, and answers with the captures it parked. Each //! group's capture comes from its leader at the apply of the request's first //! barrier. A group with no capture fails the backup with a retryable error @@ -66,10 +66,10 @@ pub(crate) async fn capture_database( database_id: database_id.as_u64(), tenants: tenants.iter().copied().collect(), }; - // The barrier names a tenant only to frame its Raft entry. + // The snapshot request names a tenant only to frame it. let framing = tenants.first().copied().unwrap_or(0); - super::super::cut::cut_with_capture(state, framing, cut, &request).await?; + super::super::cut::cut_with_capture(state, cut, &request).await?; let mut replies = take_replies(state, request.request_id); let remote: Vec = source_assignment(state)? .into_iter() @@ -130,11 +130,10 @@ pub(crate) fn take_replies(state: &SharedState, request: u64) -> Vec /// the encoded captures this node parked. pub(crate) async fn cut_and_reply( state: &SharedState, - framing: u64, watermark: u64, request: &CutCaptureRequest, ) -> Result, Error> { - super::super::cut::cut_with_capture(state, framing, watermark, request).await?; + super::super::cut::cut_with_capture(state, watermark, request).await?; zerompk::to_msgpack_vec(&take_replies(state, request.request_id)).map_err(|e| { Error::Serialization { format: "msgpack".into(), diff --git a/nodedb/src/control/backup/cut_capture/registry.rs b/nodedb/src/control/backup/cut_capture/registry.rs index 1d4e17055..9fd741ce8 100644 --- a/nodedb/src/control/backup/cut_capture/registry.rs +++ b/nodedb/src/control/backup/cut_capture/registry.rs @@ -5,8 +5,9 @@ //! //! Every replica of a group applies the same log, so every replica sees the //! same first barrier of a request in that group. Only that first barrier -//! captures. A later barrier of the same request, proposed by another node, -//! sits above entries the first barrier already cut away, and never captures. +//! captures. A later barrier of the same request, a leader's second proposal +//! of it, sits above entries the first barrier already cut away, and never +//! captures. use std::collections::HashMap; use std::sync::Mutex; diff --git a/nodedb/src/control/backup/cut_order/driver.rs b/nodedb/src/control/backup/cut_order/driver.rs new file mode 100644 index 000000000..18ebb9772 --- /dev/null +++ b/nodedb/src/control/backup/cut_order/driver.rs @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The task that places one ordered cut's barrier in every data group this +//! node leads. +//! +//! A node starts the driver when it learns of the cut: its coordinator takes +//! the cut here, or a scheduler here receives the cut's marker. The driver +//! runs until every data group this node hosts applied the cut's barrier +//! here, or the cut's window closes. While this node leads a group, the +//! driver proposes the barrier once every scheduler of the group's vShards +//! on this node passed the cut's marker: every transaction sequenced before +//! the marker then installed here, so its redo is in the log before the +//! barrier. A proposal a later term overwrote is proposed again. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use nodedb_cluster::calvin::SEQUENCER_GROUP_ID; +use nodedb_cluster::{METADATA_GROUP_ID, MultiRaft}; + +use crate::control::security::auth_fence::cluster::{hosts_group, routed_groups}; +use crate::control::state::SharedState; +use crate::control::wal_replication::ReplicatedEntry; +use crate::types::DatabaseId; + +use super::ordered_cut::{CutKey, CutWindow, OrderedCut}; +use super::registry::Proposed; + +/// How often a driver looks again when no change woke it. A leadership move +/// and a group applying past a lost proposal wake nothing. +const DRIVE_POLL: Duration = Duration::from_millis(20); + +/// The tenant a barrier entry is framed under. A barrier orders every entry +/// of its group, whatever tenant writes it. No step reads the tenant of a +/// barrier: the apply's barrier arm ignores it, and a barrier raises no +/// tenant mark. +const BARRIER_FRAME_TENANT: u64 = 0; + +/// Every data group this node hosts. +pub(crate) fn hosted_data_groups(state: &SharedState) -> Vec { + routed_groups(state) + .into_iter() + .filter(|group_id| { + *group_id != METADATA_GROUP_ID + && *group_id != SEQUENCER_GROUP_ID + && hosts_group(state, *group_id) + }) + .collect() +} + +/// Every vShard `group_id` homes, from this node's routing table. +fn group_vshards(state: &SharedState, group_id: u64) -> Vec { + state + .cluster_routing + .as_ref() + .map_or_else(Vec::new, |routing| { + routing + .read() + .unwrap_or_else(|p| p.into_inner()) + .vshards_for_group(group_id) + }) +} + +/// Start the driver of `cut` on this node, unless one runs for it. +pub(crate) fn ensure_driver(state: &Arc, cut: &OrderedCut) { + if !state.calvin.cut_barriers.claim_driver(cut) { + return; + } + let key = cut.key(); + let state = Arc::clone(state); + tokio::spawn(async move { + drive(&state, key).await; + state.calvin.cut_barriers.driver_stopped(key); + }); +} + +/// Place `key`'s barrier in every group this node leads, until every group +/// this node hosts applied it here or the cut's window closes. +async fn drive(state: &Arc, key: CutKey) { + let Some(cut) = state.calvin.cut_barriers.cut(key) else { + return; + }; + let window = CutWindow::on(state, cut.hlc); + let mut shutdown = state.shutdown.subscribe(); + loop { + // Registered before the checks, so a change between a check and the + // wait still wakes it. + let changed = state.calvin.cut_barriers.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + let open: Vec = hosted_data_groups(state) + .into_iter() + .filter(|group_id| state.calvin.cut_barriers.applied(key, *group_id).is_none()) + .collect(); + if open.is_empty() { + return; + } + // no-determinism: the window bounds only when this leader proposes. + // The barrier's place in the log is the one every replica applies. + if state.hlc_clock.now().wall_ns > window.propose_until { + for group_id in open { + report_unplaced(state, &cut, group_id); + } + return; + } + for group_id in open { + if let Err(error) = propose_if_due(state, &cut, group_id) { + tracing::error!( + group_id, + hlc = cut.hlc, + %error, + "backup cut: the cut's barrier entry cannot be built; the cut fails" + ); + report_unplaced(state, &cut, group_id); + return; + } + } + tokio::select! { + () = shutdown.wait_cancelled() => return, + () = &mut changed => {} + () = tokio::time::sleep(DRIVE_POLL) => {} + } + } +} + +/// This node's leader term of `group_id`, while it leads the group. +fn leader_term(multi_raft: &Mutex, group_id: u64) -> Option { + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .leader_term(group_id) +} + +/// Propose `cut`'s barrier into `group_id` when this node leads the group, +/// every scheduler of the group's vShards here passed the cut's marker, and +/// no proposal of the current term is still in the log. Fails only when the +/// barrier entry cannot be built. +fn propose_if_due(state: &SharedState, cut: &OrderedCut, group_id: u64) -> crate::Result<()> { + let Some(multi_raft) = state.multi_raft.get() else { + return Ok(()); + }; + let key = cut.key(); + let Some(term) = leader_term(multi_raft, group_id) else { + return Ok(()); + }; + if let Some(proposed) = state.calvin.cut_barriers.proposal(key, group_id) { + // An entry of an earlier term that the group applied past with no + // barrier of the cut is gone: a later term overwrote it. + let lost = proposed.term != term + || state.applied_index_watcher(group_id).current() >= proposed.log_index; + if !lost { + return Ok(()); + } + state + .calvin + .cut_barriers + .forget_proposal(key, group_id, proposed); + if state.calvin.cut_barriers.applied(key, group_id).is_some() { + return Ok(()); + } + } + let vshards = group_vshards(state, group_id); + if !state + .calvin + .cuts + .lagging_among(cut.hlc, &vshards) + .is_empty() + { + return Ok(()); + } + let Some(&vshard_id) = vshards.first() else { + return Ok(()); + }; + let bytes = ReplicatedEntry::new( + BARRIER_FRAME_TENANT, + DatabaseId::DEFAULT.as_u64(), + vshard_id, + cut.barrier_write(), + ) + .encode()?; + let landed = { + let mut mr = multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + // Leadership can move between the checks and the proposal. + if mr.leader_term(group_id) != Some(term) { + return Ok(()); + } + mr.propose(vshard_id, bytes) + }; + match landed { + Ok((landed_group, log_index)) => { + state + .calvin + .cut_barriers + .note_proposed(key, landed_group, Proposed { term, log_index }) + } + Err(error) => tracing::debug!( + group_id, + hlc = cut.hlc, + %error, + "backup cut: the group refused the cut's barrier; the driver proposes it again" + ), + } + Ok(()) +} + +/// Report that the cut's window closed before `group_id` applied its +/// barrier here. +fn report_unplaced(state: &SharedState, cut: &OrderedCut, group_id: u64) { + let led_here = state + .multi_raft + .get() + .is_some_and(|multi_raft| leader_term(multi_raft, group_id).is_some()); + let lagging = state + .calvin + .cuts + .lagging_among(cut.hlc, &group_vshards(state, group_id)); + tracing::warn!( + group_id, + hlc = cut.hlc, + restore_point = cut.restore_point, + led_here, + ?lagging, + "backup cut: the cut's window closed before its barrier applied in this group; \ + the backup or restore point that took the cut failed" + ); + crate::diag::cut_barrier_not_placed(group_id, cut.hlc, cut.restore_point, led_here, lagging); +} diff --git a/nodedb/src/control/backup/cut_order/marker.rs b/nodedb/src/control/backup/cut_order/marker.rs new file mode 100644 index 000000000..1600292c2 --- /dev/null +++ b/nodedb/src/control/backup/cut_order/marker.rs @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A cut's Calvin marker: proposed into the sequencer log until it took. + +use std::time::Duration; + +use nodedb_cluster::calvin::SequencerEntry; + +use crate::Error; +use crate::control::state::SharedState; + +use super::ordered_cut::OrderedCut; +use super::wait::await_placed; + +/// How long the cut waits for its Calvin marker before it proposes the +/// marker again. A leader change can drop a proposed marker. A second copy +/// is harmless: it names the same cut, and every scheduler keeps the place +/// of the first copy it received. +const CUT_MARKER_RETRY: Duration = Duration::from_secs(1); + +/// What one cut marker carries. +#[derive(Clone, Copy)] +pub(crate) struct Marker<'a> { + pub hlc: u64, + pub restore_point: u64, + /// The cut whose barriers the marker orders, `None` for a marker with no + /// barrier. + pub barrier: Option<&'a OrderedCut>, +} + +/// Propose `marker` into the sequencer log, and wait until every Calvin +/// scheduler on this node passed it, or `deadline`. +/// +/// With no scheduler here, nothing here shows that the marker applied. An +/// ordered cut then proposes the marker again until the barrier of every +/// group of `groups` applied here: the groups' leaders place the barriers +/// only once their schedulers passed the marker. +pub(crate) async fn place_marker( + state: &SharedState, + marker: Marker<'_>, + groups: &[u64], + deadline: tokio::time::Instant, +) -> Result<(), Error> { + let cuts = &state.calvin.cuts; + let calvin_here = !cuts.is_empty(); + // With no scheduler here, the barriers of this node's groups show that + // the marker applied. + let watched = marker + .barrier + .filter(|_| !calvin_here && !groups.is_empty()); + if !calvin_here && watched.is_none() { + // No Calvin scheduler runs here and no group here waits on the + // marker: this node applies no Calvin write. + return Ok(()); + } + let proposer = state + .calvin + .sequencer_proposer + .get() + .ok_or_else(|| Error::Internal { + detail: "backup: this node takes a Calvin cut, but no sequencer proposer is set, so \ + the consistent cut cannot place its marker. Retry the backup once the \ + cluster finished starting" + .into(), + })?; + let entry = zerompk::to_msgpack_vec(&SequencerEntry::CutMarker { + hlc: marker.hlc, + restore_point: marker.restore_point, + barrier: marker.barrier.map(OrderedCut::to_wire), + }) + .map_err(|error| Error::Internal { + detail: format!("backup: encode the Calvin cut marker: {error}"), + })?; + let mut last_refusal = None; + loop { + if let Err(error) = proposer.propose(entry.clone()) { + last_refusal = Some(error.to_string()); + } + let attempt_deadline = deadline.min(tokio::time::Instant::now() + CUT_MARKER_RETRY); + let lagging = cuts.await_passed(marker.hlc, attempt_deadline).await; + if lagging.is_empty() { + let Some(cut) = watched else { + return Ok(()); + }; + if await_placed(state, cut.key(), groups, attempt_deadline).await + || tokio::time::Instant::now() >= deadline + { + // A group with no barrier by the deadline fails the cut's + // barrier wait, which names it. + return Ok(()); + } + continue; + } + if tokio::time::Instant::now() >= deadline { + return Err(Error::Internal { + detail: format!( + "backup: the Calvin schedulers of vShards {lagging:?} did not pass the \ + consistent-cut marker in time, so Calvin transactions sequenced before \ + the backup may not be installed (last marker refusal: {}). Retry the \ + backup", + last_refusal.as_deref().unwrap_or("none") + ), + }); + } + } +} diff --git a/nodedb/src/control/backup/cut_order/mod.rs b/nodedb/src/control/backup/cut_order/mod.rs new file mode 100644 index 000000000..73d10b7f7 --- /dev/null +++ b/nodedb/src/control/backup/cut_order/mod.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: BUSL-1.1 + +pub mod driver; +pub mod marker; +pub mod ordered_cut; +pub mod registry; +pub mod wait; + +pub use ordered_cut::{CutKey, CutWindow, OrderedCut}; +pub use registry::CutBarriers; diff --git a/nodedb/src/control/backup/cut_order/ordered_cut.rs b/nodedb/src/control/backup/cut_order/ordered_cut.rs new file mode 100644 index 000000000..29b6fa860 --- /dev/null +++ b/nodedb/src/control/backup/cut_order/ordered_cut.rs @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! One cut whose data-group barriers are ordered with its Calvin marker. +//! +//! The cut's marker carries the barrier. In every data group, the leader +//! proposes the barrier once every scheduler of the group's vShards on that +//! node passed the marker, and holds the redo of every later transaction +//! until it applied the barrier. A transaction sequenced before the marker +//! installed before its scheduler passed it, so its redo sits before the +//! barrier. A later one proposes its redo only after the barrier. +//! +//! The ordering has a window, measured from the cut's watermark on the HLC +//! wall clock. A leader proposes the barrier only inside +//! [`CutWindow::propose_until`]. A scheduler holds later redo only until +//! [`CutWindow::hold_until`], a clock-skew margin later. A leader that +//! released its holds at the end of the window therefore never sees a new +//! leader place the barrier after the released redo, while clock skew stays +//! below the margin. A cut whose window closed with no barrier in a group +//! failed: its coordinator's deadline ends inside the window. + +use std::time::Duration; + +use nodedb_cluster::calvin::{CutBarrierWire, CutCaptureWire}; +use nodedb_physical::physical_plan::CutCaptureRequest; + +use crate::control::state::SharedState; +use crate::control::wal_replication::ReplicatedWrite; + +/// The clock skew between two nodes a cut's window tolerates. +const CLOCK_SKEW_MARGIN: Duration = Duration::from_secs(30); + +/// What names one cut on every node: its watermark, its restore point, and +/// its capture request, `0` for none. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct CutKey { + pub hlc: u64, + pub restore_point: u64, + pub capture: u64, +} + +/// A cut whose barriers the data-group leaders order with its marker. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OrderedCut { + /// The cut's watermark: HLC wall time in nanoseconds. + pub hlc: u64, + /// The cluster restore point the cut takes, `0` for a backup's cut. + pub restore_point: u64, + /// A database backup's capture request. + pub capture: Option, +} + +impl OrderedCut { + pub fn key(&self) -> CutKey { + CutKey { + hlc: self.hlc, + restore_point: self.restore_point, + capture: self + .capture + .as_ref() + .map_or(0, |request| request.request_id), + } + } + + /// The cut a marker carrying `hlc`, `restore_point` and `barrier` names. + pub fn from_marker(hlc: u64, restore_point: u64, barrier: &CutBarrierWire) -> Self { + Self { + hlc, + restore_point, + capture: barrier.capture.as_ref().map(|wire| CutCaptureRequest { + request_id: wire.request_id, + database_id: wire.database_id, + tenants: wire.tenants.clone(), + }), + } + } + + /// The barrier this cut's marker carries. + pub fn to_wire(&self) -> CutBarrierWire { + CutBarrierWire { + capture: self.capture.as_ref().map(|request| CutCaptureWire { + request_id: request.request_id, + database_id: request.database_id, + tenants: request.tenants.clone(), + }), + } + } + + /// The write of this cut's barrier entry. + pub fn barrier_write(&self) -> ReplicatedWrite { + ReplicatedWrite::CutBarrier { + hlc: self.hlc, + restore_point: self.restore_point, + capture: self.capture.clone(), + } + } +} + +/// The ordering window of one cut on this node. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CutWindow { + /// The last HLC wall time a leader proposes the cut's barrier at. + pub propose_until: u64, + /// The last HLC wall time a scheduler holds later redo at. + pub hold_until: u64, +} + +impl CutWindow { + /// The window of the cut at `hlc`, given the statement deadline + /// `deadline`. A coordinator waits one deadline from its watermark, and + /// a remote source node one more from the request it receives. + pub fn of(hlc: u64, deadline: Duration) -> Self { + let propose = deadline.saturating_mul(2); + let propose_until = hlc.saturating_add(nanos(propose)); + Self { + propose_until, + hold_until: propose_until.saturating_add(nanos(CLOCK_SKEW_MARGIN)), + } + } + + /// The window of the cut at `hlc` under `state`'s statement deadline. + pub fn on(state: &SharedState, hlc: u64) -> Self { + Self::of( + hlc, + Duration::from_secs(state.tuning.network.default_deadline_secs), + ) + } +} + +/// `duration` in nanoseconds, saturating. +fn nanos(duration: Duration) -> u64 { + u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_cut_round_trips_through_its_marker() { + let cut = OrderedCut { + hlc: 7, + restore_point: 3, + capture: Some(CutCaptureRequest { + request_id: 11, + database_id: 2, + tenants: vec![1, 4], + }), + }; + let back = OrderedCut::from_marker(7, 3, &cut.to_wire()); + assert_eq!(back, cut); + assert_eq!( + back.key(), + CutKey { + hlc: 7, + restore_point: 3, + capture: 11 + } + ); + let plain = OrderedCut { + hlc: 7, + restore_point: 0, + capture: None, + }; + assert_eq!(plain.key().capture, 0); + assert_ne!(plain.key(), cut.key(), "the capture keeps two cuts apart"); + } + + #[test] + fn the_hold_outlasts_the_proposal_window_by_the_skew_margin() { + let window = CutWindow::of(1_000, Duration::from_secs(30)); + assert_eq!(window.propose_until, 1_000 + 60_000_000_000); + assert_eq!( + window.hold_until - window.propose_until, + nanos(CLOCK_SKEW_MARGIN) + ); + } +} diff --git a/nodedb/src/control/backup/cut_order/registry.rs b/nodedb/src/control/backup/cut_order/registry.rs new file mode 100644 index 000000000..dcdb15e71 --- /dev/null +++ b/nodedb/src/control/backup/cut_order/registry.rs @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Where a node tracks each ordered cut's barrier per data group. +//! +//! For every cut it knows, the node keeps, per group: +//! +//! - where it proposed the cut's barrier while it led the group, as the +//! leader term and log index; +//! - the log index of the first barrier of the cut it applied in the group. +//! +//! A leader's schedulers hold the redo of every transaction sequenced after +//! a cut's marker until the group's barrier applied here. An applied entry is +//! committed, so every redo proposed afterwards sits after it in the log, +//! whatever term the proposer leads. +//! +//! One driver task per cut runs on a node (see [`super::driver`]). It +//! proposes the barrier of every group this node leads, and proposes it +//! again when a later term overwrote the proposal. + +use std::collections::BTreeMap; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use tokio::sync::Notify; +use tokio::sync::futures::Notified; + +use super::ordered_cut::{CutKey, OrderedCut}; + +/// How long a node remembers a cut no driver runs for. Every hold and every +/// wait on a cut ends within its window, far below this. +const CUT_TTL: Duration = Duration::from_secs(30 * 60); + +/// Where this node proposed a cut's barrier. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Proposed { + /// The group's leader term the proposal was made in. + pub term: u64, + pub log_index: u64, +} + +#[derive(Debug, Default, Clone, Copy)] +struct GroupBarrier { + proposed: Option, + applied: Option, +} + +#[derive(Debug)] +struct TrackedCut { + cut: OrderedCut, + groups: BTreeMap, + driving: bool, + touched: Instant, +} + +impl TrackedCut { + fn new(cut: &OrderedCut, now: Instant) -> Self { + Self { + cut: cut.clone(), + groups: BTreeMap::new(), + driving: false, + touched: now, + } + } +} + +/// Every ordered cut this node knows, by key. +#[derive(Debug, Default)] +pub struct CutBarriers { + cuts: Mutex>, + changed: Notify, +} + +impl CutBarriers { + pub fn new() -> Self { + Self::default() + } + + /// Track `cut` and claim its driver. `true` when no driver ran for it: + /// the caller starts one. + pub(crate) fn claim_driver(&self, cut: &OrderedCut) -> bool { + self.with_cut(cut, |tracked| { + !std::mem::replace(&mut tracked.driving, true) + }) + } + + /// The driver of `key` stopped. A later claim starts a new one. + pub(crate) fn driver_stopped(&self, key: CutKey) { + let mut cuts = self.cuts.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(tracked) = cuts.get_mut(&key) { + tracked.driving = false; + tracked.touched = Instant::now(); + } + } + + /// The cut `key` names, while this node tracks it. + pub(crate) fn cut(&self, key: CutKey) -> Option { + self.cuts + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&key) + .map(|tracked| tracked.cut.clone()) + } + + /// Note a barrier of `cut` this node applies at `log_index` of `group`. + /// The first barrier of the cut in the group keeps its index. + pub(crate) fn note_applied(&self, cut: &OrderedCut, group: u64, log_index: u64) { + self.with_cut(cut, |tracked| { + let barrier = tracked.groups.entry(group).or_default(); + barrier.applied.get_or_insert(log_index); + }); + self.changed.notify_waiters(); + } + + /// The log index of the first barrier of `key` this node applied in + /// `group`. + pub(crate) fn applied(&self, key: CutKey, group: u64) -> Option { + self.group(key, group).and_then(|barrier| barrier.applied) + } + + /// Note that this node proposed `key`'s barrier into `group` while it + /// led the group. + pub(crate) fn note_proposed(&self, key: CutKey, group: u64, proposed: Proposed) { + { + let mut cuts = self.cuts.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(tracked) = cuts.get_mut(&key) { + tracked.groups.entry(group).or_default().proposed = Some(proposed); + } + } + self.changed.notify_waiters(); + } + + /// Where this node proposed `key`'s barrier into `group`. + pub(crate) fn proposal(&self, key: CutKey, group: u64) -> Option { + self.group(key, group).and_then(|barrier| barrier.proposed) + } + + /// Forget the proposal `lost` of `key` in `group`: a later term + /// overwrote it. A newer proposal stays. + pub(crate) fn forget_proposal(&self, key: CutKey, group: u64, lost: Proposed) { + let mut cuts = self.cuts.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(barrier) = cuts + .get_mut(&key) + .and_then(|tracked| tracked.groups.get_mut(&group)) + && barrier.proposed == Some(lost) + { + barrier.proposed = None; + } + } + + /// A wait that resolves at the next change of any cut. Enable it before + /// the check it guards. + pub(crate) fn notified(&self) -> Notified<'_> { + self.changed.notified() + } + + fn group(&self, key: CutKey, group: u64) -> Option { + self.cuts + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&key) + .and_then(|tracked| tracked.groups.get(&group).copied()) + } + + /// Apply `update` to the tracked `cut`, tracking it when new. Drops every + /// cut past its TTL that no driver runs for. + fn with_cut(&self, cut: &OrderedCut, update: impl FnOnce(&mut TrackedCut) -> T) -> T { + let now = Instant::now(); + let mut cuts = self.cuts.lock().unwrap_or_else(|p| p.into_inner()); + cuts.retain(|_, tracked| { + tracked.driving || now.saturating_duration_since(tracked.touched) < CUT_TTL + }); + let tracked = cuts + .entry(cut.key()) + .or_insert_with(|| TrackedCut::new(cut, now)); + tracked.touched = now; + update(tracked) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cut(hlc: u64) -> OrderedCut { + OrderedCut { + hlc, + restore_point: 0, + capture: None, + } + } + + #[test] + fn one_driver_runs_per_cut() { + let barriers = CutBarriers::new(); + assert!(barriers.claim_driver(&cut(5))); + assert!(!barriers.claim_driver(&cut(5)), "a copy finds the driver"); + assert!(barriers.claim_driver(&cut(6)), "another cut has its own"); + barriers.driver_stopped(cut(5).key()); + assert!( + barriers.claim_driver(&cut(5)), + "a stopped driver starts again" + ); + } + + #[test] + fn the_first_applied_barrier_keeps_its_index() { + let barriers = CutBarriers::new(); + let key = cut(5).key(); + assert_eq!(barriers.applied(key, 1), None); + barriers.note_applied(&cut(5), 1, 40); + barriers.note_applied(&cut(5), 1, 52); + assert_eq!(barriers.applied(key, 1), Some(40)); + assert_eq!(barriers.applied(key, 2), None, "a barrier binds its group"); + assert_eq!(barriers.cut(key), Some(cut(5))); + } + + /// A lost proposal is forgotten only when it is the one recorded. + #[test] + fn a_lost_proposal_is_forgotten_once() { + let barriers = CutBarriers::new(); + let key = cut(5).key(); + barriers.claim_driver(&cut(5)); + assert_eq!(barriers.proposal(key, 1), None); + + let proposed = Proposed { + term: 3, + log_index: 17, + }; + barriers.note_proposed(key, 1, proposed); + assert_eq!(barriers.proposal(key, 1), Some(proposed)); + assert_eq!(barriers.proposal(key, 2), None, "another group"); + + barriers.forget_proposal( + key, + 1, + Proposed { + term: 2, + log_index: 9, + }, + ); + assert_eq!( + barriers.proposal(key, 1), + Some(proposed), + "an older loss keeps the newer proposal" + ); + barriers.forget_proposal(key, 1, proposed); + assert_eq!(barriers.proposal(key, 1), None); + } + + #[tokio::test] + async fn an_applied_barrier_wakes_a_waiter() { + let barriers = std::sync::Arc::new(CutBarriers::new()); + let waiter = std::sync::Arc::clone(&barriers); + let woken = tokio::spawn(async move { + let notified = waiter.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if waiter.applied(cut(5).key(), 1).is_none() { + notified.await; + } + waiter.applied(cut(5).key(), 1) + }); + tokio::task::yield_now().await; + barriers.note_applied(&cut(5), 1, 8); + let applied = tokio::time::timeout(Duration::from_secs(5), woken) + .await + .expect("the waiter wakes") + .expect("the waiter runs"); + assert_eq!(applied, Some(8)); + } +} diff --git a/nodedb/src/control/backup/cut_order/wait.rs b/nodedb/src/control/backup/cut_order/wait.rs new file mode 100644 index 000000000..20959e8a9 --- /dev/null +++ b/nodedb/src/control/backup/cut_order/wait.rs @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A cut's wait on this node for the barrier of each data group it hosts. + +use std::time::Duration; + +use crate::Error; +use crate::control::security::auth_fence::cluster::{hosts_group, wait_applied}; +use crate::control::state::SharedState; + +use super::ordered_cut::{CutKey, OrderedCut}; + +/// How often a wait looks again when no change woke it. A group this node +/// leaves wakes nothing. +const WAIT_POLL: Duration = Duration::from_millis(50); + +/// Wait until every group of `groups` applied `cut`'s barrier here, and +/// every entry before the barrier applied here, or `deadline`. A group this +/// node left needs no wait: the source node that snapshots it takes its own +/// cut. +pub(crate) async fn await_group_barriers( + state: &SharedState, + cut: &OrderedCut, + groups: &[u64], + deadline: tokio::time::Instant, +) -> Result<(), Error> { + let waits = futures::future::join_all( + groups + .iter() + .map(|group_id| await_group_barrier(state, cut, *group_id, deadline)), + ) + .await; + waits.into_iter().collect() +} + +/// Wait until every group of `groups` applied the barrier of `key` here, or +/// `until`. A group this node left counts as placed. Returns whether every +/// group's barrier applied. +pub(crate) async fn await_placed( + state: &SharedState, + key: CutKey, + groups: &[u64], + until: tokio::time::Instant, +) -> bool { + loop { + let changed = state.calvin.cut_barriers.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + let placed = groups.iter().all(|group_id| { + state.calvin.cut_barriers.applied(key, *group_id).is_some() + || !hosts_group(state, *group_id) + }); + if placed { + return true; + } + if tokio::time::Instant::now() >= until { + return false; + } + tokio::select! { + () = &mut changed => {} + () = tokio::time::sleep(WAIT_POLL) => {} + () = tokio::time::sleep_until(until) => {} + } + } +} + +async fn await_group_barrier( + state: &SharedState, + cut: &OrderedCut, + group_id: u64, + deadline: tokio::time::Instant, +) -> Result<(), Error> { + let key = cut.key(); + loop { + // Registered before the checks, so a change between a check and the + // wait still wakes it. + let changed = state.calvin.cut_barriers.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + if let Some(barrier_index) = state.calvin.cut_barriers.applied(key, group_id) { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + return match wait_applied(state, group_id, barrier_index, remaining).await { + Ok(()) => Ok(()), + Err(error) if !hosts_group(state, group_id) => { + note_left(group_id, &error); + Ok(()) + } + Err(error) => Err(Error::Internal { + detail: format!( + "backup: the consistent-cut barrier of raft group {group_id} at log \ + index {barrier_index} did not apply on this node: {error}. Retry the \ + backup" + ), + }), + }; + } + if !hosts_group(state, group_id) { + note_left(group_id, &"no barrier applied yet"); + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(unplaced_error(state, cut, group_id)); + } + tokio::select! { + () = &mut changed => {} + () = tokio::time::sleep(WAIT_POLL) => {} + () = tokio::time::sleep_until(deadline) => {} + } + } +} + +fn note_left(group_id: u64, why: &dyn std::fmt::Display) { + tracing::info!( + group_id, + %why, + "backup: this node left the group before its cut barrier applied here; the group \ + needs no cut on this node" + ); +} + +/// The error of a group whose barrier did not apply here by the deadline. +fn unplaced_error(state: &SharedState, cut: &OrderedCut, group_id: u64) -> Error { + let leads = state.multi_raft.get().is_some_and(|multi_raft| { + multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .leader_term(group_id) + .is_some() + }); + let place = if leads { + let vshards = state + .cluster_routing + .as_ref() + .map_or_else(Vec::new, |routing| { + routing + .read() + .unwrap_or_else(|p| p.into_inner()) + .vshards_for_group(group_id) + }); + let lagging = state.calvin.cuts.lagging_among(cut.hlc, &vshards); + format!( + "This node leads the group. The Calvin schedulers of its vShards {lagging:?} here \ + have not passed the cut's marker" + ) + } else { + "Another node leads the group and places the barrier once its Calvin schedulers \ + passed the cut's marker" + .to_owned() + }; + Error::Internal { + detail: format!( + "backup: raft group {group_id} applied no consistent-cut barrier at watermark {} on \ + this node in time. {place}. Retry the backup", + cut.hlc + ), + } +} diff --git a/nodedb/src/control/backup/mod.rs b/nodedb/src/control/backup/mod.rs index d266eaab2..245c80885 100644 --- a/nodedb/src/control/backup/mod.rs +++ b/nodedb/src/control/backup/mod.rs @@ -4,6 +4,7 @@ pub mod bind_capture; pub mod capture; pub mod cut; pub mod cut_capture; +pub mod cut_order; pub mod database; pub mod detect; pub mod metadata; diff --git a/nodedb/src/control/backup/orchestrator.rs b/nodedb/src/control/backup/orchestrator.rs index 1ebd1d5db..04d2b4c1d 100644 --- a/nodedb/src/control/backup/orchestrator.rs +++ b/nodedb/src/control/backup/orchestrator.rs @@ -52,7 +52,7 @@ pub async fn backup_tenant(state: &Arc, tenant_id: u64) -> Result, +} + +/// The cuts one scheduler holds later redo for, in arrival order. +#[derive(Debug, Default)] +pub struct CutHolds { + held: Vec, +} + +impl CutHolds { + /// Hold for `cut`, whose marker arrived after epochs up to `through`. + /// Returns `false` for a copy of a cut already held. + pub fn receive(&mut self, cut: OrderedCut, through: Option) -> bool { + let key = cut.key(); + if self.held.iter().any(|held| held.cut.key() == key) { + return false; + } + self.held.push(HeldCut { cut, through }); + true + } + + /// Whether a transaction of `epoch` waits for a held cut's barrier. + pub fn holds(&self, epoch: u64) -> bool { + self.held + .iter() + .any(|held| held.through.is_none_or(|through| through < epoch)) + } + + /// Stop holding for every cut `ended` names. Returns whether any ended. + pub fn end(&mut self, mut ended: impl FnMut(&OrderedCut) -> bool) -> bool { + let before = self.held.len(); + self.held.retain(|held| !ended(&held.cut)); + self.held.len() != before + } + + /// Every held cut, in arrival order. + pub fn cuts(&self) -> impl Iterator { + self.held.iter().map(|held| &held.cut) + } + + pub fn is_empty(&self) -> bool { + self.held.is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cut(hlc: u64) -> OrderedCut { + OrderedCut { + hlc, + restore_point: 0, + capture: None, + } + } + + #[test] + fn only_a_transaction_after_the_marker_waits() { + let mut holds = CutHolds::default(); + assert!(!holds.holds(9), "no cut, no hold"); + assert!(holds.receive(cut(100), Some(7))); + assert!(!holds.holds(7), "epoch 7 came before the marker"); + assert!(!holds.holds(3)); + assert!(holds.holds(8), "epoch 8 came after it"); + } + + #[test] + fn a_marker_before_any_epoch_holds_every_transaction() { + let mut holds = CutHolds::default(); + holds.receive(cut(100), None); + assert!(holds.holds(0)); + assert!(holds.holds(1)); + } + + /// A second copy of a held cut arrives later and keeps the first copy's + /// place: the transactions between the two copies still wait. + #[test] + fn a_copy_of_a_held_cut_keeps_the_first_place() { + let mut holds = CutHolds::default(); + assert!(holds.receive(cut(100), Some(7))); + assert!( + !holds.receive(cut(100), Some(12)), + "a copy is not a new cut" + ); + assert!(holds.holds(9), "epoch 9 came after the first copy"); + assert_eq!(holds.cuts().count(), 1); + } + + #[test] + fn an_ended_cut_releases_its_transactions() { + let mut holds = CutHolds::default(); + holds.receive(cut(100), Some(7)); + holds.receive(cut(200), Some(10)); + assert!(!holds.end(|_| false)); + assert!(holds.end(|ended| ended.hlc == 100)); + assert!(!holds.holds(9), "only the later cut holds, from epoch 11"); + assert!(holds.holds(11)); + assert!(holds.end(|_| true)); + assert!(holds.is_empty()); + assert!(!holds.holds(11)); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs index 92ec0d9d7..d4b954665 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/cut_marker.rs @@ -9,19 +9,40 @@ //! delivered after the marker records a commit HLC above the marker's //! watermark, so a restore of that backup refuses it. The data-group apply //! raises the tenant's write mark from the redo entry's commit HLC. +//! +//! A marker that carries a barrier orders the data groups too (see +//! [`crate::control::backup::cut_order`]). The scheduler holds the redo of +//! every transaction delivered after it until this node applied the cut's +//! barrier in the vShard's group, or the cut's window closed. It starts the +//! node's driver of the cut, which places the barrier while this node leads +//! the group. + +use std::sync::Arc; + +use nodedb_cluster::calvin::CutBarrierWire; use super::scheduler::Scheduler; +use crate::control::backup::cut_order::driver::ensure_driver; +use crate::control::backup::cut_order::{CutWindow, OrderedCut}; +use crate::control::state::SharedState; impl Scheduler { - /// Receive a backup's cut marker carrying `hlc`, in input order. + /// Receive a backup's cut marker carrying `hlc`, in input order. A + /// `barrier` orders the cut's barriers with the marker. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn receive_cut_marker( &mut self, hlc: u64, + restore_point: u64, + barrier: Option>, ) { - if self - .cut_floors - .receive(hlc, self.applied.highest_seen_epoch()) - { + let highest_seen = self.applied.highest_seen_epoch(); + if let Some(barrier) = barrier { + self.hold_for_cut( + OrderedCut::from_marker(hlc, restore_point, &barrier), + highest_seen, + ); + } + if self.cut_floors.receive(hlc, highest_seen) { self.shared.calvin.cuts.note_passed(self.vshard_id, hlc); } self.report_passed_cuts(); @@ -41,4 +62,242 @@ impl Scheduler { } self.cut_floors.fold(self.applied.fully_applied_epoch()); } + + /// Hold the redo of every transaction after the marker of `cut`, which + /// arrived after epochs up to `through`, and start the cut's driver. + fn hold_for_cut(&mut self, cut: OrderedCut, through: Option) { + if window_closed(&self.shared, &cut) { + // A marker this scheduler receives again after a restart, long + // after its cut ended. Its cut placed every barrier it ever will. + return; + } + if self.cut_holds.receive(cut.clone(), through) { + ensure_driver(&self.shared, &cut); + } + } + + /// Whether the redo of a transaction of `epoch` waits for an ordered + /// cut's barrier. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn held_for_cut( + &self, + epoch: u64, + ) -> bool { + self.cut_holds.holds(epoch) + } + + /// End the hold of every cut whose barrier this node applied in the + /// vShard's group, or whose window closed. Start the driver again of a + /// cut still held whose driver stopped. Returns whether a hold ended: the + /// held redo can then be proposed. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn release_cut_holds( + &mut self, + ) -> bool { + if self.cut_holds.is_empty() { + return false; + } + let group_id = self.data_group(); + let shared = Arc::clone(&self.shared); + let vshard_id = self.vshard_id; + let ended = self.cut_holds.end(|cut| { + let applied = group_id.is_some_and(|group_id| { + shared + .calvin + .cut_barriers + .applied(cut.key(), group_id) + .is_some() + }); + if applied { + return true; + } + let closed = window_closed(&shared, cut); + if closed { + tracing::warn!( + vshard_id, + ?group_id, + hlc = cut.hlc, + "calvin scheduler: a cut's window closed before its barrier applied here; \ + releasing the redo held for it" + ); + } + closed + }); + for cut in self.cut_holds.cuts() { + // no-determinism: the window bounds only when this node proposes. + if shared.hlc_clock.now().wall_ns <= CutWindow::on(&shared, cut.hlc).propose_until { + ensure_driver(&shared, cut); + } + } + ended + } + + /// The data group that homes this vShard. + fn data_group(&self) -> Option { + let mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + mr.routing() + .read() + .unwrap_or_else(|p| p.into_inner()) + .group_for_vshard(self.vshard_id) + .ok() + } +} + +/// Whether `cut`'s hold window closed on `state`'s clock. +/// +/// no-determinism: the window bounds only how long this node holds redo. +/// Every replica installs the redo at its place in the log. +fn window_closed(state: &SharedState, cut: &OrderedCut) -> bool { + state.hlc_clock.now().wall_ns > CutWindow::on(state, cut.hlc).hold_until +} + +#[cfg(test)] +mod tests { + use nodedb_cluster::calvin::types::SchedulerInput; + + use super::*; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler, make_validate_only_txn, test_coll_vshard, + }; + + /// A cut whose watermark is now, so its window is open. + fn fresh_cut(scheduler: &Scheduler) -> OrderedCut { + OrderedCut { + hlc: scheduler.shared.hlc_clock.now().wall_ns, + restore_point: 0, + capture: None, + } + } + + fn marker(cut: &OrderedCut) -> SchedulerInput { + SchedulerInput::CutMarker { + hlc: cut.hlc, + restore_point: cut.restore_point, + barrier: Some(Arc::new(cut.to_wire())), + } + } + + /// Only a transaction delivered after a barrier-carrying marker waits, + /// and the marker still passes once the earlier ones finished. + #[tokio::test] + async fn a_marker_with_a_barrier_holds_only_later_redo() { + let (mut scheduler, _dir) = build_test_scheduler(test_coll_vshard()); + scheduler + .process_scheduler_input(SchedulerInput::Txn(Box::new(make_validate_only_txn(4, 0)))); + let cut = fresh_cut(&scheduler); + scheduler.process_scheduler_input(marker(&cut)); + + assert!(!scheduler.held_for_cut(4), "epoch 4 came before the marker"); + assert!(scheduler.held_for_cut(5), "epoch 5 came after it"); + assert!( + scheduler + .shared + .calvin + .cut_barriers + .cut(cut.key()) + .is_some(), + "the cut's driver tracks it" + ); + } + + /// The hold ends once this node applied the cut's barrier in the + /// vShard's group, and not before. A barrier of another group ends + /// nothing. + #[tokio::test] + async fn the_hold_ends_when_the_group_applied_the_barrier() { + let (mut scheduler, _dir) = build_test_scheduler(test_coll_vshard()); + let cut = fresh_cut(&scheduler); + scheduler.process_scheduler_input(marker(&cut)); + assert!( + scheduler.held_for_cut(0), + "a marker before any epoch holds all" + ); + let group_id = scheduler + .data_group() + .expect("the fixture routes every vShard"); + + assert!(!scheduler.release_cut_holds()); + scheduler + .shared + .calvin + .cut_barriers + .note_applied(&cut, group_id.wrapping_add(1), 9); + assert!(!scheduler.release_cut_holds(), "another group's barrier"); + assert!(scheduler.held_for_cut(0)); + + scheduler + .shared + .calvin + .cut_barriers + .note_applied(&cut, group_id, 12); + assert!(scheduler.release_cut_holds()); + assert!(!scheduler.held_for_cut(0)); + assert!(!scheduler.release_cut_holds(), "nothing left to end"); + } + + /// A second copy of the marker keeps the first copy's place, and a copy + /// of a cut whose barrier applied holds nothing for long. + #[tokio::test] + async fn a_marker_copy_keeps_the_first_place() { + let (mut scheduler, _dir) = build_test_scheduler(test_coll_vshard()); + scheduler + .process_scheduler_input(SchedulerInput::Txn(Box::new(make_validate_only_txn(4, 0)))); + let cut = fresh_cut(&scheduler); + scheduler.process_scheduler_input(marker(&cut)); + scheduler + .process_scheduler_input(SchedulerInput::Txn(Box::new(make_validate_only_txn(6, 0)))); + scheduler.process_scheduler_input(marker(&cut)); + assert!( + scheduler.held_for_cut(5), + "epoch 5 came after the first copy, so it waits" + ); + assert_eq!(scheduler.cut_holds.cuts().count(), 1); + + let group_id = scheduler + .data_group() + .expect("the fixture routes every vShard"); + scheduler + .shared + .calvin + .cut_barriers + .note_applied(&cut, group_id, 30); + assert!(scheduler.release_cut_holds()); + scheduler.process_scheduler_input(marker(&cut)); + assert!( + scheduler.release_cut_holds(), + "a late copy finds the barrier applied" + ); + assert!(!scheduler.held_for_cut(7)); + } + + /// A marker received long after its cut's window closed holds nothing: + /// a replay after a restart never blocks redo on a cut that ended. + #[tokio::test] + async fn a_stale_marker_holds_nothing() { + let (mut scheduler, _dir) = build_test_scheduler(test_coll_vshard()); + let stale = OrderedCut { + hlc: 1, + restore_point: 0, + capture: None, + }; + scheduler.process_scheduler_input(marker(&stale)); + assert!(!scheduler.held_for_cut(0)); + assert!(scheduler.cut_holds.is_empty()); + } + + /// A cut whose window closes with no barrier ends its hold. + #[tokio::test] + async fn a_closed_window_ends_the_hold() { + let (mut scheduler, _dir) = build_test_scheduler(test_coll_vshard()); + let cut = fresh_cut(&scheduler); + scheduler.process_scheduler_input(marker(&cut)); + assert!(scheduler.held_for_cut(1)); + let past_window = CutWindow::on(&scheduler.shared, cut.hlc) + .hold_until + .saturating_add(1); + scheduler + .shared + .hlc_clock + .update(nodedb_types::Hlc::new(past_window, 0)); + assert!(scheduler.release_cut_holds()); + assert!(!scheduler.held_for_cut(1)); + } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs index a8d24a437..16fd6f776 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs @@ -64,7 +64,11 @@ impl Scheduler { SchedulerInput::Txn(txn) => self.process_or_hold_for_metadata(*txn), SchedulerInput::Reserve { owner, key } => self.install_reservation(owner, key), SchedulerInput::Release { owner, reason } => self.release_reservation(owner, reason), - SchedulerInput::CutMarker { hlc } => self.receive_cut_marker(hlc), + SchedulerInput::CutMarker { + hlc, + restore_point, + barrier, + } => self.receive_cut_marker(hlc, restore_point, barrier), SchedulerInput::TxnPart { txn, index, diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs index a40add2c9..b1fd6d0c5 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs @@ -20,6 +20,7 @@ //! | Proposed, then the group applied past the entry with no install | Propose again. | //! | A transient refusal | Propose again on the stall tick. | //! | The entry cannot be built | Halt: the verdict is COMMIT, so the txn cannot drop. | +//! | Sequenced after an ordered cut's marker whose barrier this node has not applied | Hold. Propose once the barrier applied. | //! //! Every copy that commits installs at most once: the apply loop claims the //! position in the vShard's applied ledger. @@ -75,6 +76,13 @@ impl Scheduler { if self.redo_propose_held(txn_id) { return; } + // A txn sequenced after an ordered cut's marker proposes its redo + // only once this node applied the cut's barrier in the vShard's + // group, so the redo lands after the barrier. The pass that ends the + // hold proposes it again. + if self.held_for_cut(txn_id.epoch) { + return; + } // A snapshot install replaced this vShard's base since this // scheduler started. The next scheduler starts from the installed // state and stages the txn again. @@ -430,6 +438,61 @@ mod tests { } } + /// A slice sequenced after an ordered cut's marker proposes nothing until + /// this node applied the cut's barrier in the vShard's group. Then the + /// pass that ends the hold proposes it. + #[tokio::test] + async fn a_slice_after_a_cut_marker_waits_for_the_barrier() { + use crate::control::backup::cut_order::OrderedCut; + use nodedb_cluster::calvin::types::SchedulerInput; + + let txn_id = TxnId::new(14, 2); + let (mut scheduler, _dir, _data_side) = resolving_leader(txn_id); + let cut = OrderedCut { + hlc: scheduler.shared.hlc_clock.now().wall_ns, + restore_point: 0, + capture: None, + }; + scheduler.process_scheduler_input(SchedulerInput::CutMarker { + hlc: cut.hlc, + restore_point: 0, + barrier: Some(std::sync::Arc::new(cut.to_wire())), + }); + + scheduler.finish_redo_resolve(txn_id, resolved_answer()); + assert_eq!(proposed_at(&scheduler, txn_id), None, "the redo is held"); + scheduler.retry_redo_proposals(); + assert_eq!( + proposed_at(&scheduler, txn_id), + None, + "the stall tick keeps it held" + ); + + let group_id = { + let mr = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()); + mr.routing() + .read() + .unwrap_or_else(|p| p.into_inner()) + .group_for_vshard(VSHARD) + .expect("the fixture routes every vShard") + }; + scheduler + .shared + .calvin + .cut_barriers + .note_applied(&cut, group_id, 1); + assert!(scheduler.release_cut_holds()); + scheduler.retry_redo_proposals(); + assert!( + proposed_at(&scheduler, txn_id).is_some(), + "the barrier applied, so the redo is proposed" + ); + assert!(!scheduler.is_apply_halted()); + } + /// A proposal refused because this node no longer leads demotes the /// scheduler: the txn is held `Following` for the next leader. #[tokio::test] diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs index d3ca26bd6..c51fc702c 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs @@ -36,6 +36,8 @@ impl Scheduler { let capacity_freed = Arc::clone(&self.capacity_freed); // Woken when the data-group apply loop concludes a stamped redo. let inbox = Arc::clone(self.inbox.inbox()); + // Woken when a cut's barrier is proposed or applied on this node. + let shared = Arc::clone(&self.shared); // Set when a tick left armed catch-up unreplayed. The next open-gate // pass fires the tick at once to resume it. let mut catch_up_resume = false; @@ -50,11 +52,19 @@ impl Scheduler { if self.resends_deferred() { self.redispatch_deferred(); } + // Registered before the hold check below, for the same reason. + let cut_changed = shared.calvin.cut_barriers.notified(); + tokio::pin!(cut_changed); + cut_changed.as_mut().enable(); // Every pass reads the role first: a promotion stages held txns, // a demotion stops staging. self.refresh_role(); self.drain_inbox(); + // A cut's barrier applied here: propose the redo it held. + if self.release_cut_holds() { + self.retry_redo_proposals(); + } self.check_dependent_barrier_timeouts(); self.check_awaiting_verdict_stalls(); self.restage_due(); @@ -145,6 +155,11 @@ impl Scheduler { // applied, and stages the held txns once it has. } + _ = &mut cut_changed, if !self.cut_holds.is_empty() => { + // The next loop pass ends the hold of every cut whose + // barrier applied here. + } + _ = &mut capacity_notified, if self.resends_deferred() => { // Capacity freed: the next loop pass re-sends deferred // requests in FIFO order. diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs index 4f8c4df2c..94189c3b5 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs @@ -133,6 +133,9 @@ pub struct Scheduler { /// set and the markers not yet reported. pub(in crate::control::cluster::calvin::scheduler::driver::core) cut_floors: crate::control::cluster::calvin::scheduler::cut_floor::CutFloors, + /// Ordered cuts whose barrier this vShard's later redo waits for. + pub(in crate::control::cluster::calvin::scheduler::driver::core) cut_holds: + crate::control::cluster::calvin::scheduler::cut_hold::CutHolds, /// This vShard's applied ledger: the gate's seed, and the shared record /// of each position this scheduler finishes. pub(in crate::control::cluster::calvin::scheduler::driver::core) ledger: @@ -310,6 +313,7 @@ impl Scheduler { read_result_rx, applied: AppliedGate::new(fully_applied_epoch, applied_tail), cut_floors: Default::default(), + cut_holds: Default::default(), ledger, caught_up, rebuild_target_epoch, diff --git a/nodedb/src/control/cluster/calvin/scheduler/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/mod.rs index a36daf75d..290645f11 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/mod.rs @@ -4,6 +4,7 @@ pub mod applied_gate; pub mod applied_ledger; pub mod caught_up; pub mod cut_floor; +pub mod cut_hold; pub mod driver; pub mod inbox; pub mod lock; diff --git a/nodedb/src/control/cluster/calvin_snapshot/capture.rs b/nodedb/src/control/cluster/calvin_snapshot/capture.rs index 24b66f15b..36dc18404 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/capture.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/capture.rs @@ -68,6 +68,7 @@ pub async fn await_calvin_cut( let marker = zerompk::to_msgpack_vec(&SequencerEntry::CutMarker { hlc, restore_point: 0, + barrier: None, }) .map_err(|error| Error::Internal { detail: format!("snapshot build: group {group_id}: encode the Calvin cut marker: {error}"), diff --git a/nodedb/src/control/distributed_applier/apply_loop/cut_barrier.rs b/nodedb/src/control/distributed_applier/apply_loop/cut_barrier.rs new file mode 100644 index 000000000..486595688 --- /dev/null +++ b/nodedb/src/control/distributed_applier/apply_loop/cut_barrier.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A backup cut's barrier entry: the group's floor above the cut, the +//! restore point's place, and a database backup's capture. +//! +//! The apply notes each barrier in the node's cut registry in log order, at +//! the entry's start. A cut waiting on this node reads where the group's +//! barrier sits, and a leader's schedulers release the Calvin redo they held +//! for the cut. + +use crate::control::array_sync::raft_apply::AppliedPosition; +use crate::control::backup::cut_order::OrderedCut; +use crate::control::state::SharedState; +use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite}; + +use super::context::{ApplyContext, FinishedApply}; +use super::lane::QueuedEntry; +use super::proposal_gate::EntryOutcome; +use super::start::Prepared; + +/// Where a cut barrier cuts its group. +#[derive(Clone, Copy)] +struct CutBarrierPoint { + hlc: u64, + restore_point: u64, + log_term: u64, +} + +/// Prepare the barrier of `cut` at `pos`, whose entry has term `log_term`. +pub(super) fn prepare_cut_barrier<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + log_term: u64, + cut: OrderedCut, +) -> Prepared<'a> { + ctx.state + .calvin + .cut_barriers + .note_applied(&cut, pos.group_id, pos.log_index); + let barrier = CutBarrierPoint { + hlc: cut.hlc, + restore_point: cut.restore_point, + log_term, + }; + match cut.capture { + Some(request) => prepare_capture_barrier(ctx, pos, barrier, request), + None => prepare_plain_barrier(ctx, pos, barrier), + } +} + +/// Note a cut barrier an installed snapshot covers. The entry is committed, +/// and the snapshot holds every entry before it, so a cut waiting here and a +/// leader's held redo read the barrier as applied. +pub(super) fn note_covered_barrier(state: &SharedState, group_id: u64, queued: &QueuedEntry) { + if let Some(ReplicatedEntry { + write: + ReplicatedWrite::CutBarrier { + hlc, + restore_point, + capture, + }, + .. + }) = &queued.decoded + { + let cut = OrderedCut { + hlc: *hlc, + restore_point: *restore_point, + capture: capture.clone(), + }; + state + .calvin + .cut_barriers + .note_applied(&cut, group_id, queued.entry.index); + } +} + +/// Prepare a backup capture's cut barrier. The lane starts a barrier with +/// nothing else of its group in flight, and starts no later entry until it +/// finishes: the floor is durable, and the capture holds every entry at or +/// below it and none above. +fn prepare_capture_barrier<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + barrier: CutBarrierPoint, + request: nodedb_physical::physical_plan::CutCaptureRequest, +) -> Prepared<'a> { + let AppliedPosition { + group_id, + log_index, + applied_key, + .. + } = pos; + let CutBarrierPoint { + hlc, + restore_point, + log_term, + } = barrier; + Prepared::Exclusive(Box::pin(async move { + persist_floor_durably(ctx, group_id, log_index, hlc).await; + record_restore_point(ctx, group_id, restore_point, hlc, log_index, log_term); + crate::control::backup::cut_capture::apply::capture_at_barrier( + ctx.state, group_id, &request, + ) + .await; + finish_barrier(ctx, group_id, log_index, applied_key) + })) +} + +/// Prepare a cut barrier with no capture. A failed floor write holds the +/// group until the floor is durable. +fn prepare_plain_barrier<'a>( + ctx: ApplyContext<'a>, + pos: AppliedPosition, + barrier: CutBarrierPoint, +) -> Prepared<'a> { + let AppliedPosition { + group_id, + log_index, + applied_key, + .. + } = pos; + let CutBarrierPoint { + hlc, + restore_point, + log_term, + } = barrier; + let persisted = + crate::control::pitr::restore_point::persist_cut_floor(ctx.state, group_id, log_index, hlc); + if persisted.is_ok() { + record_restore_point(ctx, group_id, restore_point, hlc, log_index, log_term); + return Prepared::Barrier; + } + Prepared::Exclusive(Box::pin(async move { + persist_floor_durably(ctx, group_id, log_index, hlc).await; + record_restore_point(ctx, group_id, restore_point, hlc, log_index, log_term); + finish_barrier(ctx, group_id, log_index, applied_key) + })) +} + +/// Record the group's place at the cluster restore point `restore_point` a +/// cut barrier takes. `0` takes none. +fn record_restore_point( + ctx: ApplyContext<'_>, + group_id: u64, + restore_point: u64, + hlc: u64, + log_index: u64, + log_term: u64, +) { + if restore_point == 0 { + return; + } + crate::control::pitr::restore_point::record_group_point( + ctx.state, + nodedb_wal::record::RestorePointPayload { + id: restore_point, + hlc, + group_id, + applied_index: log_index, + term: log_term, + next_epoch: 0, + epoch_system_ms: 0, + vshards: Vec::new(), + }, + ); +} + +/// Persist the floor of the barrier at `log_index`, retrying until it holds. +/// A floor write that keeps failing wedges the node until it succeeds. +async fn persist_floor_durably(ctx: ApplyContext<'_>, group_id: u64, log_index: u64, hlc: u64) { + crate::control::pitr::restore_point::persist_until_durable( + &ctx.state.metadata_apply_wedge, + group_id, + log_index, + || { + crate::control::pitr::restore_point::persist_cut_floor( + ctx.state, group_id, log_index, hlc, + ) + }, + ) + .await; +} + +/// Resolve a barrier's waiter once every step of the barrier is durable. +fn finish_barrier( + ctx: ApplyContext<'_>, + group_id: u64, + log_index: u64, + applied_key: u64, +) -> FinishedApply { + ctx.tracker.complete( + group_id, + log_index, + applied_key, + Ok(crate::control::distributed_applier::AppliedWrite::unversioned(Vec::new())), + ); + FinishedApply { + group_id, + log_index, + outcome: EntryOutcome::Skipped, + } +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/mod.rs b/nodedb/src/control/distributed_applier/apply_loop/mod.rs index a10a41bca..2423e526f 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/mod.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/mod.rs @@ -19,6 +19,8 @@ //! - [`pipeline`]: every group's lane and the applies that run. //! - [`lane`]: one group's queued and started entries, settled in log order. //! - [`start`]: prepares one entry and routes it to its apply path. +//! - [`cut_barrier`]: a backup cut's barrier: the group's cut floor, the +//! restore point's place, and a database backup's capture. //! - [`context`]: the handles an apply borrows, and the futures the loop //! collects. //! - [`calvin_read_result`]: forwards a committed `CalvinReadResult` entry to @@ -52,6 +54,7 @@ mod calvin_read_result; mod calvin_redo; mod collection_route; mod context; +mod cut_barrier; mod driver; mod group_watch; mod helpers; diff --git a/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs b/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs index bfe988027..033bc6937 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs @@ -211,6 +211,7 @@ impl<'a> Pipeline<'a> { // At or below the snapshot's Raft index: the installed state // and the Raft boundary both hold the entry. EntryAdmission::Covered => { + super::cut_barrier::note_covered_barrier(shared, group_id, &queued); if !self.conclude_covered( group_id, log_index, @@ -226,6 +227,7 @@ impl<'a> Pipeline<'a> { // applying and extends the durable prefix, so a restart never // delivers it again. EntryAdmission::CoveredByCut => { + super::cut_barrier::note_covered_barrier(shared, group_id, &queued); if !self.conclude_covered( group_id, log_index, diff --git a/nodedb/src/control/distributed_applier/apply_loop/start.rs b/nodedb/src/control/distributed_applier/apply_loop/start.rs index 7a0fd0089..f20dd6cfd 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/start.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/start.rs @@ -14,6 +14,7 @@ use crate::control::array_sync::raft_apply::{ }; use nodedb_raft::message::LogEntry; +use crate::control::backup::cut_order::OrderedCut; use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite}; use crate::types::{DatabaseId, TenantId}; @@ -259,15 +260,16 @@ fn prepare_replicated<'a>( // Every entry after the barrier records above the cut, on this // life and on every later one. watch.raise_cut(pos.group_id, pos.log_index, hlc); - let barrier = CutBarrierPoint { - hlc, - restore_point, + super::cut_barrier::prepare_cut_barrier( + ctx, + pos, log_term, - }; - match capture { - Some(request) => prepare_capture_barrier(ctx, pos, barrier, request), - None => prepare_plain_barrier(ctx, pos, barrier), - } + OrderedCut { + hlc, + restore_point, + capture, + }, + ) } ReplicatedWrite::CalvinReadResult { epoch, @@ -358,138 +360,3 @@ fn prepare_array_op<'a>( } })) } - -/// Where a cut barrier cuts its group. -#[derive(Clone, Copy)] -struct CutBarrierPoint { - hlc: u64, - restore_point: u64, - log_term: u64, -} - -/// Prepare a backup capture's cut barrier. The lane starts a barrier with -/// nothing else of its group in flight, and starts no later entry until it -/// finishes: the floor is durable, and the capture holds every entry at or -/// below it and none above. -fn prepare_capture_barrier<'a>( - ctx: ApplyContext<'a>, - pos: AppliedPosition, - barrier: CutBarrierPoint, - request: nodedb_physical::physical_plan::CutCaptureRequest, -) -> Prepared<'a> { - let AppliedPosition { - group_id, - log_index, - applied_key, - .. - } = pos; - let CutBarrierPoint { - hlc, - restore_point, - log_term, - } = barrier; - Prepared::Exclusive(Box::pin(async move { - persist_floor_durably(ctx, group_id, log_index, hlc).await; - record_restore_point(ctx, group_id, restore_point, hlc, log_index, log_term); - crate::control::backup::cut_capture::apply::capture_at_barrier( - ctx.state, group_id, &request, - ) - .await; - finish_barrier(ctx, group_id, log_index, applied_key) - })) -} - -/// Prepare a cut barrier with no capture. A failed floor write holds the -/// group until the floor is durable. -fn prepare_plain_barrier<'a>( - ctx: ApplyContext<'a>, - pos: AppliedPosition, - barrier: CutBarrierPoint, -) -> Prepared<'a> { - let AppliedPosition { - group_id, - log_index, - applied_key, - .. - } = pos; - let CutBarrierPoint { - hlc, - restore_point, - log_term, - } = barrier; - let persisted = - crate::control::pitr::restore_point::persist_cut_floor(ctx.state, group_id, log_index, hlc); - if persisted.is_ok() { - record_restore_point(ctx, group_id, restore_point, hlc, log_index, log_term); - return Prepared::Barrier; - } - Prepared::Exclusive(Box::pin(async move { - persist_floor_durably(ctx, group_id, log_index, hlc).await; - record_restore_point(ctx, group_id, restore_point, hlc, log_index, log_term); - finish_barrier(ctx, group_id, log_index, applied_key) - })) -} - -/// Record the group's place at the cluster restore point `restore_point` a -/// cut barrier takes. `0` takes none. -fn record_restore_point( - ctx: ApplyContext<'_>, - group_id: u64, - restore_point: u64, - hlc: u64, - log_index: u64, - log_term: u64, -) { - if restore_point == 0 { - return; - } - crate::control::pitr::restore_point::record_group_point( - ctx.state, - nodedb_wal::record::RestorePointPayload { - id: restore_point, - hlc, - group_id, - applied_index: log_index, - term: log_term, - next_epoch: 0, - epoch_system_ms: 0, - vshards: Vec::new(), - }, - ); -} - -/// Persist the floor of the barrier at `log_index`, retrying until it holds. -/// A floor write that keeps failing wedges the node until it succeeds. -async fn persist_floor_durably(ctx: ApplyContext<'_>, group_id: u64, log_index: u64, hlc: u64) { - crate::control::pitr::restore_point::persist_until_durable( - &ctx.state.metadata_apply_wedge, - group_id, - log_index, - || { - crate::control::pitr::restore_point::persist_cut_floor( - ctx.state, group_id, log_index, hlc, - ) - }, - ) - .await; -} - -/// Resolve a barrier's waiter once every step of the barrier is durable. -fn finish_barrier( - ctx: ApplyContext<'_>, - group_id: u64, - log_index: u64, - applied_key: u64, -) -> FinishedApply { - ctx.tracker.complete( - group_id, - log_index, - applied_key, - Ok(crate::control::distributed_applier::AppliedWrite::unversioned(Vec::new())), - ); - FinishedApply { - group_id, - log_index, - outcome: EntryOutcome::Skipped, - } -} diff --git a/nodedb/src/control/exec_receiver/backup_cut.rs b/nodedb/src/control/exec_receiver/backup_cut.rs index 88dbac18d..49e451395 100644 --- a/nodedb/src/control/exec_receiver/backup_cut.rs +++ b/nodedb/src/control/exec_receiver/backup_cut.rs @@ -35,7 +35,7 @@ pub(super) async fn take_backup_cut( else { return Ok(plan); }; - crate::control::backup::cut::cut_at(state, tenant_id, watermark) + crate::control::backup::cut::cut_at(state, watermark) .await .map_err(execution_error_to_typed)?; Ok(PhysicalPlan::Meta(MetaOp::CreateTenantSnapshot { @@ -65,7 +65,6 @@ pub(super) async fn answer_capture_plan( plan: &PhysicalPlan, ) -> Option { let PhysicalPlan::Meta(MetaOp::CreateTenantSnapshot { - tenant_id, cut_watermark, cut_capture: Some(request), .. @@ -79,10 +78,9 @@ pub(super) async fn answer_capture_plan( message: "a cut capture request carries no cut watermark".into(), })); }; - let answer = crate::control::backup::cut_capture::collect::cut_and_reply( - state, *tenant_id, watermark, request, - ) - .await; + let answer = + crate::control::backup::cut_capture::collect::cut_and_reply(state, watermark, request) + .await; Some(match answer { Ok(payload) => ExecuteResponse::ok(vec![payload], 0, 0), Err(error) => ExecuteResponse::err(execution_error_to_typed(error)), diff --git a/nodedb/src/control/pitr/restore_point/record.rs b/nodedb/src/control/pitr/restore_point/record.rs index cbffeb47f..db4cf11c5 100644 --- a/nodedb/src/control/pitr/restore_point/record.rs +++ b/nodedb/src/control/pitr/restore_point/record.rs @@ -7,8 +7,11 @@ //! data group it hosts, plus the Calvin sequencer, at the point's watermark. //! Every replica of a group applies the cut barrier at the same log index and //! records it, so the group's place at the point is that index on every node. -//! A group can hold more than one barrier for a point: every replica proposes -//! its own. The lowest index is the group's place. +//! The group's leader places the barrier (see +//! `crate::control::backup::cut_order`). A group can still hold more than one +//! barrier for a point: a leader proposes it again when a later term +//! overwrote its proposal, and the overwritten copy can commit too. The +//! lowest index is the group's place. use std::sync::{Arc, Weak}; @@ -21,12 +24,6 @@ use tracing::{error, info, warn}; use crate::control::state::SharedState; use crate::wal::manager::NO_APPLY_KEY; -/// Tenant the restore point's cut barriers are framed under. A barrier -/// orders every entry of its group, whatever tenant writes it. No step reads -/// the tenant of a barrier: the proposal stamps and forwards opaque bytes, -/// the apply's barrier arms ignore it, and a barrier raises no tenant mark. -const CUT_FRAME_TENANT: u64 = 0; - /// Record one group's place at a restore point in this node's WAL, durably, /// and move this node's clock past the point's watermark. A data group's /// record lists the vShards it homes now. A failure is logged: a cluster @@ -107,7 +104,7 @@ pub fn sequencer_hook(shared: Weak) -> RestorePointHook { /// point's records is then sealed, so the archiver uploads it. pub fn spawn_node_cut(state: Arc, id: u64, hlc: u64) { tokio::spawn(async move { - match crate::control::backup::cut::cut_at_point(&state, CUT_FRAME_TENANT, hlc, id).await { + match crate::control::backup::cut::cut_at_point(&state, hlc, id).await { Ok(()) => info!(restore_point = id, "restore point cut taken on this node"), Err(e) => warn!( restore_point = id, diff --git a/nodedb/src/control/server/exchange/all_cores/read_cut.rs b/nodedb/src/control/server/exchange/all_cores/read_cut.rs index 54a7bd1d8..a66df9898 100644 --- a/nodedb/src/control/server/exchange/all_cores/read_cut.rs +++ b/nodedb/src/control/server/exchange/all_cores/read_cut.rs @@ -76,6 +76,7 @@ async fn resolve_read_cut(state: &SharedState, marker: u64) -> crate::Result Vec { + let registered = self.registered.lock().unwrap_or_else(|p| p.into_inner()); + let recent = self.recent.lock().unwrap_or_else(|p| p.into_inner()); + let passed = recent.get(&hlc).map(|marker| &marker.passed); + vshards + .iter() + .copied() + .filter(|vshard_id| registered.contains(vshard_id)) + .filter(|vshard_id| passed.is_none_or(|passed| !passed.contains(vshard_id))) + .collect() + } + /// Wait until every scheduler passed the marker `hlc`, or `deadline`. /// Returns the vShards still lagging, empty once every one passed. pub async fn await_passed(&self, hlc: u64, deadline: tokio::time::Instant) -> Vec { @@ -310,4 +324,24 @@ mod tests { let other: BTreeSet = [3].into_iter().collect(); assert_eq!(cuts.await_vshards_passed(80, &other, soon()).await, None); } + + /// A group's barrier waits only on the schedulers of its vShards that + /// run here, and only until each passed exactly the cut's marker. + #[test] + fn a_group_lags_on_its_own_registered_vshards() { + let cuts = CalvinCuts::default(); + cuts.register(1); + cuts.register(2); + assert_eq!(cuts.lagging_among(90, &[1, 2, 9]), vec![1, 2]); + cuts.note_passed(1, 90); + cuts.note_passed(2, 91); + assert_eq!( + cuts.lagging_among(90, &[1, 2, 9]), + vec![2], + "vShard 2 passed another marker; vShard 9 runs no scheduler here" + ); + cuts.note_passed(2, 90); + assert!(cuts.lagging_among(90, &[1, 2, 9]).is_empty()); + assert!(cuts.lagging_among(95, &[]).is_empty(), "no vShard, no wait"); + } } diff --git a/nodedb/src/control/state/calvin_local.rs b/nodedb/src/control/state/calvin_local.rs index 495106d2e..0d7ee5d29 100644 --- a/nodedb/src/control/state/calvin_local.rs +++ b/nodedb/src/control/state/calvin_local.rs @@ -81,6 +81,9 @@ pub struct CalvinLocalState { pub admission_holds: AdmissionHolds, /// The backup cut markers each local scheduler passed. pub cuts: CalvinCuts, + /// Each ordered cut's barrier per data group: where this node proposed + /// it while it led the group, and where this node applied it. + pub cut_barriers: crate::control::backup::cut_order::CutBarriers, /// Where this node's Calvin state of each vShard is whole from. pub bases: CalvinBases, /// Hands this node's sequencer entries to the sequencer Raft group. Set @@ -117,6 +120,7 @@ impl CalvinLocalState { autocommit_lock_seq: AtomicU32::new(0), admission_holds: AdmissionHolds::new(), cuts: CalvinCuts::default(), + cut_barriers: crate::control::backup::cut_order::CutBarriers::new(), bases: CalvinBases::default(), sequencer_proposer: OnceLock::new(), applied: CalvinAppliedLedgers::default(), diff --git a/nodedb/src/diag/context/cut_barrier.rs b/nodedb/src/diag/context/cut_barrier.rs new file mode 100644 index 000000000..4d464fd68 --- /dev/null +++ b/nodedb/src/diag/context/cut_barrier.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Forensic payloads for a backup cut's data-group barriers. + +use faultbox::DomainContext; +use faultbox::serde_json::{Value, json}; + +/// A cut's barrier driver stopped with a data group of this node that never +/// applied the cut's barrier. +pub(in crate::diag) struct CutBarrierNotPlaced { + pub group_id: u64, + /// The cut's watermark. + pub hlc: u64, + pub restore_point: u64, + /// Whether this node led the group when the driver stopped. + pub led_here: bool, + /// The vShards of the group whose scheduler here had not passed the + /// cut's marker. + pub lagging: Vec, +} + +impl DomainContext for CutBarrierNotPlaced { + fn domain_kind(&self) -> &'static str { + "nodedb.cut_barrier_not_placed" + } + + fn grouping_key(&self) -> String { + // One report per group: the watermark and the lagging vShards + // change with every cut. + format!("cut_barrier_not_placed:{}", self.group_id) + } + + fn to_json(&self) -> Value { + json!({ + "group_id": self.group_id, + "hlc": self.hlc, + "restore_point": self.restore_point, + "led_here": self.led_here, + "lagging_vshards": self.lagging, + "effect": "the cut has no barrier in this group. The backup or restore point \ + that took it failed. Calvin redo the leader held for the cut was \ + released without a barrier", + "operator_action": "check the group's leadership and the lagging vShards' \ + Calvin schedulers: a halted scheduler never passes a cut \ + marker. Retry the backup or restore point once they run", + }) + } +} diff --git a/nodedb/src/diag/context/mod.rs b/nodedb/src/diag/context/mod.rs index d11059891..54cdf3cd0 100644 --- a/nodedb/src/diag/context/mod.rs +++ b/nodedb/src/diag/context/mod.rs @@ -11,6 +11,7 @@ mod columnar; mod continuous_agg; mod crdt; mod crdt_dead_letter_store; +mod cut_barrier; mod data_plane; mod event_image; mod index_rebuild; @@ -36,6 +37,7 @@ pub(in crate::diag) use crdt::{CrdtDeadLetterNotEnqueued, HistoryCompactionNotAp pub(in crate::diag) use crdt_dead_letter_store::{ CrdtDeadLetterNotRestored, CrdtDeadLetterNotStored, }; +pub(in crate::diag) use cut_barrier::CutBarrierNotPlaced; pub use data_plane::LostResponseWrite; pub(in crate::diag) use data_plane::{ CalvinApplyHalted, CalvinCompletionTimeout, CoreFailStopped, DataPlaneResponseLost, diff --git a/nodedb/src/diag/mod.rs b/nodedb/src/diag/mod.rs index 7d8dcae58..e2768b9bf 100644 --- a/nodedb/src/diag/mod.rs +++ b/nodedb/src/diag/mod.rs @@ -14,8 +14,8 @@ pub use recording::{ calvin_completion_timeout, catalog_apply_orphan_row, collection_purge_row_missing, columnar_segment_corrupt, consumer_group_offsets_retained, continuous_aggregate_not_applied, crdt_dead_letter_not_enqueued, crdt_dead_letter_not_restored, crdt_dead_letter_not_stored, - data_plane_core_fail_stopped, data_plane_response_lost, data_plane_responses_lost, - descriptor_lease_not_renewed, entry_kind, fts_index_update_failed, + cut_barrier_not_placed, data_plane_core_fail_stopped, data_plane_response_lost, + data_plane_responses_lost, descriptor_lease_not_renewed, entry_kind, fts_index_update_failed, history_compaction_not_applied, ilp_invalid_utf8_drop, ilp_line_read_drop, index_rebuild_not_installed, metadata_apply_wedged, orphaned_index_entry_after_delete, quota_row_invalid, quota_row_undecodable, quota_row_write_failed, quota_scope_purge_incomplete, diff --git a/nodedb/src/diag/recording/cut_barrier.rs b/nodedb/src/diag/recording/cut_barrier.rs new file mode 100644 index 000000000..fdf0812a4 --- /dev/null +++ b/nodedb/src/diag/recording/cut_barrier.rs @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Capture sites of a backup cut's data-group barriers. + +use faultbox::{Capture, EventKind}; + +use crate::diag::context; + +/// Report that a cut's barrier driver stopped before `group_id` applied the +/// cut's barrier on this node. Called only from the driver, the one site +/// that sees the cut's ordering window close. +pub fn cut_barrier_not_placed( + group_id: u64, + hlc: u64, + restore_point: u64, + led_here: bool, + lagging: Vec, +) { + let ctx = context::CutBarrierNotPlaced { + group_id, + hlc, + restore_point, + led_here, + lagging, + }; + let _ = Capture::new( + EventKind::Error, + "backup cut barrier not placed in a data group", + ) + .domain(&ctx) + .emit(); +} diff --git a/nodedb/src/diag/recording/mod.rs b/nodedb/src/diag/recording/mod.rs index ac1d2a388..6f1c6ac8c 100644 --- a/nodedb/src/diag/recording/mod.rs +++ b/nodedb/src/diag/recording/mod.rs @@ -12,6 +12,7 @@ mod catalog; mod columnar; mod continuous_agg; mod crdt; +mod cut_barrier; mod data_plane; mod event_image; mod index_rebuild; @@ -37,6 +38,7 @@ pub use crdt::{ crdt_dead_letter_not_enqueued, crdt_dead_letter_not_restored, crdt_dead_letter_not_stored, history_compaction_not_applied, }; +pub use cut_barrier::cut_barrier_not_placed; pub use data_plane::{ calvin_apply_halted, calvin_completion_timeout, data_plane_core_fail_stopped, data_plane_response_lost, data_plane_responses_lost, From 5efcaf076b09fcbb6b1060d555169ceb8e783a15 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sat, 10 Oct 2026 20:58:36 +0800 Subject: [PATCH 14/19] feat(array): spread tile keys over the top bits Add tile_prefix and cell_tile_prefix, which scale each tile index over its dimension's tile count before encoding the space-filling-curve key. Tiles of one array now route to distinct buckets instead of clustering in the low bits. Export tiles_per_dim for callers that need the count. --- nodedb-array/src/lib.rs | 3 +- nodedb-array/src/tile/layout.rs | 183 +++++++++++++++++- nodedb-array/src/tile/mod.rs | 4 +- .../{homed_reads.rs => read_validation.rs} | 0 4 files changed, 184 insertions(+), 6 deletions(-) rename nodedb/src/control/server/shared/session/commit/{homed_reads.rs => read_validation.rs} (100%) diff --git a/nodedb-array/src/lib.rs b/nodedb-array/src/lib.rs index 0e49d0148..4a4d0daaa 100644 --- a/nodedb-array/src/lib.rs +++ b/nodedb-array/src/lib.rs @@ -36,7 +36,8 @@ pub use sync::{ SnapshotHeader, SnapshotSink, TileSnapshot, }; pub use tile::{ - AttrStats, DenseTile, SparseTile, TileMBR, tile_id_for_cell, tile_indices_for_cell, + AttrStats, DenseTile, SparseTile, TileMBR, cell_tile_prefix, tile_id_for_cell, + tile_indices_for_cell, tile_prefix, tiles_per_dim, }; pub use types::{ArrayId, CellValue, Coord, Domain, TileId}; pub use wal::ArrayWalRecord; diff --git a/nodedb-array/src/tile/layout.rs b/nodedb-array/src/tile/layout.rs index 6c1ff722e..cb61577d3 100644 --- a/nodedb-array/src/tile/layout.rs +++ b/nodedb-array/src/tile/layout.rs @@ -8,10 +8,10 @@ //! same tile share a [`TileId::hilbert_prefix`]. use crate::coord::encode::encode_tile_prefix_with_order; -use crate::coord::normalize::bits_per_dim; +use crate::coord::normalize::{bits_per_dim, normalize_coord}; use crate::error::{ArrayError, ArrayResult}; use crate::schema::ArraySchema; -use crate::schema::dim_spec::DimType; +use crate::schema::dim_spec::{DimSpec, DimType}; use crate::types::TileId; use crate::types::coord::value::CoordValue; use crate::types::domain::DomainBound; @@ -76,11 +76,93 @@ pub fn tile_id_for_cell( system_from_ms: i64, ) -> ArrayResult { let tile_indices = tile_indices_for_cell(schema, coord)?; - let bits = bits_per_dim(schema.arity()); - let prefix = encode_tile_prefix_with_order(schema, &tile_indices, bits)?; + let prefix = tile_prefix(schema, &tile_indices)?; Ok(TileId::new(prefix, system_from_ms)) } +/// The space-filling-curve key of the tile at `tile_indices`. +/// +/// The memtable, the segment index, the transaction overlay and retention +/// group cells by this key, and a cluster routes a tile by its top bits. +/// Every cell inside a group stays addressed by its full coordinate: a +/// lookup matches the coordinate within the group +/// (`TileBuffer::get_cell_bytes`, `extract_cell_bytes`), and a tile's MBR +/// is built from the cells it holds. Two tiles that share a key therefore +/// share one group and lose no cell. +/// +/// A dimension whose tile count fits its share of the key bits scales each +/// index over that count to the full bit width. Its tiles keep distinct, +/// ordered keys that spread over the top bits. A dimension with more tiles +/// than its bits can number, such as one with the default full-range +/// domain, keeps the low bits of the index instead. Nearby tiles then keep +/// distinct keys, and tiles `2^bits` apart share one. +pub fn tile_prefix(schema: &ArraySchema, tile_indices: &[u64]) -> ArrayResult { + if tile_indices.len() != schema.arity() { + return Err(ArrayError::CoordArityMismatch { + array: schema.name.clone(), + expected: schema.arity(), + got: tile_indices.len(), + }); + } + let bits = bits_per_dim(schema.arity()); + let scaled: Vec = tile_indices + .iter() + .zip(schema.dims.iter().zip(schema.tile_extents.iter())) + .map(|(index, (dim, extent))| scale_tile_index(*index, tiles_per_dim(dim, *extent), bits)) + .collect(); + encode_tile_prefix_with_order(schema, &scaled, bits) +} + +/// The key of the tile that holds the cell at `coord`: the key every +/// cluster path routes the cell by. Fails when `coord` lies outside the +/// schema's domain. +pub fn cell_tile_prefix(schema: &ArraySchema, coord: &[CoordValue]) -> ArrayResult { + normalize_coord(schema, coord, bits_per_dim(schema.arity()))?; + tile_prefix(schema, &tile_indices_for_cell(schema, coord)?) +} + +/// The number of tiles along `dim` for tile extent `extent`. Mirrors how +/// [`tile_indices_for_cell`] numbers them: a cell in the domain has a tile +/// index below this count. +pub fn tiles_per_dim(dim: &DimSpec, extent: u64) -> u128 { + let extent = u128::from(extent.max(1)); + match (dim.dtype, &dim.domain.lo, &dim.domain.hi) { + (DimType::Int64, DomainBound::Int64(lo), DomainBound::Int64(hi)) + | (DimType::TimestampMs, DomainBound::TimestampMs(lo), DomainBound::TimestampMs(hi)) => { + let span = ((*hi as i128) - (*lo as i128)).max(0) as u128; + span / extent + 1 + } + (DimType::Float64, DomainBound::Float64(lo), DomainBound::Float64(hi)) => { + if hi.is_finite() && lo.is_finite() && hi > lo { + u128::from((hi - lo) as u64) / extent + 1 + } else { + 1 + } + } + (DimType::String, DomainBound::String(_), DomainBound::String(_)) => extent, + _ => 1, + } +} + +/// Map tile index `index` of `tiles` tiles to `[0, 2^bits)`. +/// +/// With `tiles <= 2^bits` the index is scaled over the tile count, so +/// distinct indices keep distinct, ordered values. An index past the last +/// tile, a cell above the domain, takes the top value. With more tiles the +/// index keeps its low `bits` bits. +fn scale_tile_index(index: u64, tiles: u128, bits: u32) -> u64 { + let limit: u128 = 1u128 << bits.min(64); + let max = limit - 1; + if tiles > limit { + return u64::try_from(u128::from(index) & max).unwrap_or(u64::MAX); + } + if tiles <= 1 { + return if index == 0 { 0 } else { max as u64 }; + } + let scaled = (u128::from(index) * max / (tiles - 1)).min(max); + u64::try_from(scaled).unwrap_or(u64::MAX) +} + fn out_of_domain(schema: &ArraySchema, dim: &str, detail: &str) -> ArrayResult { Err(ArrayError::CoordOutOfDomain { array: schema.name.clone(), @@ -138,6 +220,99 @@ mod tests { assert_ne!(t1.hilbert_prefix, t2.hilbert_prefix); } + #[test] + fn tile_keys_spread_over_the_top_bits() { + let s = schema(); + let tops: std::collections::BTreeSet = (0..10) + .flat_map(|x| (0..10).map(move |y| (x * 10, y * 10))) + .map(|(x, y)| { + tile_id_for_cell(&s, &[CoordValue::Int64(x), CoordValue::Int64(y)], 0) + .unwrap() + .hilbert_prefix + >> 56 + }) + .collect(); + assert!( + tops.len() > 1, + "the tiles of one array route to more than one bucket: {tops:?}" + ); + } + + #[test] + fn every_tile_keeps_a_distinct_key() { + let s = schema(); + let keys: std::collections::BTreeSet = (0..10) + .flat_map(|x| (0..10).map(move |y| (x * 10, y * 10))) + .map(|(x, y)| { + tile_id_for_cell(&s, &[CoordValue::Int64(x), CoordValue::Int64(y)], 0) + .unwrap() + .hilbert_prefix + }) + .collect(); + assert_eq!(keys.len(), 100); + } + + #[test] + fn a_cell_routes_by_its_tile_key() { + let s = schema(); + let coord = [CoordValue::Int64(37), CoordValue::Int64(81)]; + assert_eq!( + cell_tile_prefix(&s, &coord).unwrap(), + tile_id_for_cell(&s, &coord, 0).unwrap().hilbert_prefix + ); + assert_eq!( + cell_tile_prefix(&s, &[CoordValue::Int64(30), CoordValue::Int64(89)]).unwrap(), + cell_tile_prefix(&s, &coord).unwrap(), + "cells of one tile route together" + ); + } + + #[test] + fn a_cell_outside_the_domain_has_no_route() { + let s = schema(); + assert!(cell_tile_prefix(&s, &[CoordValue::Int64(100), CoordValue::Int64(0)]).is_err()); + } + + #[test] + fn a_cell_above_the_domain_takes_the_last_key() { + let s = schema(); + let above = tile_id_for_cell(&s, &[CoordValue::Int64(150), CoordValue::Int64(0)], 0) + .unwrap() + .hilbert_prefix; + let last = tile_id_for_cell(&s, &[CoordValue::Int64(99), CoordValue::Int64(0)], 0) + .unwrap() + .hilbert_prefix; + assert_eq!(above, last); + } + + #[test] + fn nearby_tiles_of_a_full_range_dimension_keep_distinct_keys() { + let s = ArraySchemaBuilder::new("open") + .dim(DimSpec::new( + "x", + DimType::Int64, + Domain::new(DomainBound::Int64(i64::MIN), DomainBound::Int64(i64::MAX)), + )) + .dim(DimSpec::new( + "y", + DimType::Int64, + Domain::new(DomainBound::Int64(i64::MIN), DomainBound::Int64(i64::MAX)), + )) + .attr(AttrSpec::new("v", AttrType::Int64, false)) + .tile_extents(vec![10, 10]) + .build() + .unwrap(); + let keys: std::collections::BTreeSet = (0..10) + .flat_map(|x| (0..10).map(move |y| (x * 10, y * 10))) + .map(|(x, y)| { + tile_id_for_cell(&s, &[CoordValue::Int64(x), CoordValue::Int64(y)], 0) + .unwrap() + .hilbert_prefix + }) + .collect(); + assert_eq!(keys.len(), 100, "a schema with any domain builds"); + } + #[test] fn system_from_ms_propagates() { let s = schema(); diff --git a/nodedb-array/src/tile/mod.rs b/nodedb-array/src/tile/mod.rs index 9ef2272fc..f2a285abf 100644 --- a/nodedb-array/src/tile/mod.rs +++ b/nodedb-array/src/tile/mod.rs @@ -11,6 +11,8 @@ pub use cell_payload::{ is_cell_gdpr_erasure, is_cell_sentinel, is_cell_tombstone, }; pub use dense_tile::DenseTile; -pub use layout::{tile_id_for_cell, tile_indices_for_cell}; +pub use layout::{ + cell_tile_prefix, tile_id_for_cell, tile_indices_for_cell, tile_prefix, tiles_per_dim, +}; pub use mbr::{AttrStats, TileMBR}; pub use sparse_tile::SparseTile; diff --git a/nodedb/src/control/server/shared/session/commit/homed_reads.rs b/nodedb/src/control/server/shared/session/commit/read_validation.rs similarity index 100% rename from nodedb/src/control/server/shared/session/commit/homed_reads.rs rename to nodedb/src/control/server/shared/session/commit/read_validation.rs From b3fef1ec63c22ad7e53af03046353c998af86e17 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sat, 10 Oct 2026 20:58:37 +0800 Subject: [PATCH 15/19] feat(calvin): validate reads against data-group write versions Replace the per-node LSN watermark on versioned reads with a WriteVersion: the epoch and log index of the data-group entry that wrote a row, or a local LSN for writes that apply no entry. Every replica of a vShard holds the same versions, so any replica validates a read, and the served_by node pin is no longer needed. Array slice and aggregate legs report the versions they observed, and the coordinator merges them into the session read-set. Snapshot installs carry a version floor so versions stay comparable across a leader move. Read validation, read splitting and pending shard reads replace the homed, served and graph read trackers in the session commit path. --- .../cases/calvin_cluster_array_read_set.rs | 142 ++ .../cases/calvin_read_version_leader_move.rs | 211 +++ .../cases/cross_node_read_occ_abort.rs | 17 +- .../cases/gather_join_in_txn_occ.rs | 8 +- .../tests/common_suite/cases/mod.rs | 2 + .../cases/native_gather_join_in_txn_occ.rs | 12 +- .../cases/shuffle_aggregate_in_txn_occ.rs | 24 +- .../cases/shuffle_join_in_txn_occ.rs | 8 +- .../misc_suite/cases/transport_security.rs | 2 +- .../cases/wire_version_handshake.rs | 4 +- .../calvin_read_failover_cross_node.rs | 78 +- nodedb-cluster/src/bootstrap/join.rs | 2 +- .../src/calvin/types/transaction.rs | 14 +- .../src/distributed_array/coordinator/read.rs | 32 +- .../src/distributed_array/handler.rs | 44 +- .../src/distributed_array/local_executor.rs | 5 + nodedb-cluster/src/distributed_array/merge.rs | 64 + nodedb-cluster/src/distributed_array/wire.rs | 7 + nodedb-cluster/src/forward.rs | 13 +- .../raft_loop/handle_rpc/shuffle_calvin.rs | 2 +- nodedb-cluster/src/raft_loop/hooks.rs | 6 +- nodedb-cluster/src/rpc_codec/execute/codec.rs | 37 +- nodedb-cluster/src/rpc_codec/execute/types.rs | 38 +- nodedb-cluster/src/rpc_codec/shuffle.rs | 30 +- .../src/transport/client/transport.rs | 4 +- nodedb-cluster/src/transport/rpc_handler.rs | 4 +- nodedb-cluster/src/transport/server.rs | 8 +- nodedb-physical/src/physical_plan/meta.rs | 6 + .../src/physical_plan/meta_home.rs | 8 +- .../node/inspect/core_placement.rs | 1 + .../src/cluster_harness/node/inspect/crdt.rs | 2 + .../cluster_harness/node/inspect/snapshot.rs | 3 + .../node/inspect/timeseries.rs | 1 + nodedb-test-support/src/tx_batch_helpers.rs | 1 + nodedb-types/src/calvin.rs | 24 +- nodedb-types/src/lib.rs | 2 + nodedb-types/src/write_version.rs | 164 ++ nodedb/src/bridge/admission_chokepoint.rs | 1 + nodedb/src/bridge/dispatch/drain.rs | 2 +- nodedb/src/bridge/dispatch/enqueue.rs | 4 +- nodedb/src/bridge/dispatch/response_poll.rs | 10 +- nodedb/src/bridge/dispatch/test_requests.rs | 2 + nodedb/src/bridge/envelope/request.rs | 9 + nodedb/src/bridge/envelope/response.rs | 21 +- nodedb/src/control/array_catalog/ddl.rs | 2 +- .../control/array_sync/raft_apply/common.rs | 13 +- nodedb/src/control/backup/restore/durable.rs | 4 +- .../post_apply/async_dispatch/array.rs | 2 +- .../post_apply/async_dispatch/core_fanout.rs | 3 +- .../post_apply/async_dispatch/crdt_compact.rs | 2 +- nodedb/src/control/checkpoint_manager.rs | 1 + .../cluster/array_cluster_exec/executor.rs | 31 +- .../cluster/array_executor/executor.rs | 1 + .../control/cluster/array_executor/read.rs | 5 + .../control/cluster/array_executor/refusal.rs | 2 +- .../control/cluster/array_executor/write.rs | 2 +- .../driver/core/commit_resolve/vote.rs | 21 - .../calvin/scheduler/driver/core/request.rs | 1 + .../scheduler/driver/core/staged_vote.rs | 2 +- .../scheduler/driver/core/test_support.rs | 6 +- .../src/control/cluster/snapshot_applier.rs | 4 +- .../cluster/snapshot_install/dispatch.rs | 1 + .../control/cluster/snapshot_install/mod.rs | 2 + .../cluster/snapshot_install/version_floor.rs | 32 + .../cluster/start_raft/proposer_wiring.rs | 13 +- .../cluster/start_raft/sequencer_route.rs | 15 +- nodedb/src/control/crdt_admission.rs | 16 +- .../apply_loop/calvin_redo.rs | 4 +- .../apply_loop/write_dispatch.rs | 10 +- .../distributed_applier/proposal_ledger.rs | 7 +- .../propose_tracker/applied_write.rs | 47 +- .../propose_tracker/tracker.rs | 10 +- .../src/control/exec_receiver/backup_cut.rs | 2 +- nodedb/src/control/exec_receiver/executor.rs | 19 +- .../control/exec_receiver/metadata_applied.rs | 2 +- .../control/exec_receiver/stream_events.rs | 2 +- .../control/exec_receiver/surrogate_binds.rs | 4 +- .../src/control/exec_receiver/tenant_marks.rs | 2 +- nodedb/src/control/gateway/core.rs | 56 +- nodedb/src/control/gateway/dispatch_local.rs | 22 +- nodedb/src/control/gateway/dispatch_remote.rs | 64 +- nodedb/src/control/gateway/dispatcher.rs | 28 +- .../error_map/system_dispatch_refusal.rs | 2 +- nodedb/src/control/gateway/outcome.rs | 66 +- .../src/control/insert_select/orchestrator.rs | 2 +- nodedb/src/control/local_dispatch/collect.rs | 8 +- .../control/local_dispatch/error_status.rs | 9 +- .../src/control/local_dispatch/local_read.rs | 1 + nodedb/src/control/local_dispatch/mod.rs | 2 +- .../clone_materializer/dispatch.rs | 1 + nodedb/src/control/orchestrated_write.rs | 9 +- nodedb/src/control/planner/calvin/dispatch.rs | 22 +- .../control/planner/calvin/dispatch_multi.rs | 6 +- .../control/planner/calvin/submit/local.rs | 2 +- .../calvin/tx_class/dependent_builder.rs | 4 +- .../control/planner/calvin/tx_class/shared.rs | 14 +- .../planner/calvin/tx_class/static_builder.rs | 37 +- .../planner/materialized_sum/predicate.rs | 3 +- .../control/planner/materialized_sum/recon.rs | 53 +- .../planner/materialized_sum/resolve.rs | 3 +- .../planner/materialized_sum/settle.rs | 43 +- .../planner/materialized_sum/stored.rs | 11 +- .../procedural/executor/core/dispatch.rs | 2 +- .../sql_plan_convert/array_convert/dml.rs | 8 +- .../array_fn_convert/slice.rs | 223 ++- nodedb/src/control/request_tracker.rs | 4 +- nodedb/src/control/server/broadcast.rs | 6 +- .../control/server/dispatch_utils/dispatch.rs | 4 +- .../server/dispatch_utils/durable_write.rs | 16 +- .../server/dispatch_utils/minted/owned.rs | 14 +- .../server/dispatch_utils/minted/resolve.rs | 2 +- .../server/dispatch_utils/owner_read.rs | 122 +- .../submit_write/funnel/answer.rs | 2 +- .../submit_write/funnel/dispatch.rs | 4 + .../submit_write/funnel/driver.rs | 5 + .../submit_write/funnel/response.rs | 2 +- .../exchange/all_cores/base_snapshot.rs | 2 +- .../control/server/exchange/all_cores/bsp.rs | 3 +- .../server/exchange/all_cores/dispatch.rs | 27 +- .../exchange/all_cores/home_versions.rs | 8 +- .../server/exchange/all_cores/snapshot.rs | 3 +- .../control/server/exchange/all_cores/wcc.rs | 3 +- .../control/server/exchange/cluster_leaf.rs | 2 +- .../control/server/exchange/core_outcome.rs | 15 +- nodedb/src/control/server/exchange/gather.rs | 37 +- .../control/server/exchange/owning_core.rs | 4 +- .../src/control/server/exchange/received.rs | 3 +- .../server/exchange/resolve/capture.rs | 14 +- .../exchange/resolve/exchange/gather_arm.rs | 10 +- .../resolve/exchange/post_process_arm.rs | 2 +- .../server/exchange/resolve/join_input.rs | 4 +- .../control/server/exchange/resolve/peers.rs | 14 +- .../server/exchange/resolve/shuffle.rs | 26 +- .../exchange/resolve/shuffle_aggregate.rs | 20 +- .../src/control/server/exchange/response.rs | 32 +- .../src/control/server/graph_dispatch/bfs.rs | 9 +- .../graph_dispatch/bsp_pagerank/coord.rs | 9 +- .../graph_dispatch/bsp_pagerank/scatter.rs | 6 +- .../server/graph_dispatch/bsp_wcc/coord.rs | 9 +- .../server/graph_dispatch/bsp_wcc/scatter.rs | 6 +- .../server/graph_dispatch/cluster_resolve.rs | 19 +- .../control/server/graph_dispatch/helpers.rs | 2 +- .../src/control/server/graph_dispatch/hop.rs | 28 +- .../server/graph_dispatch/keyed_read.rs | 6 +- .../server/graph_dispatch/match_broadcast.rs | 8 +- .../graph_dispatch/match_scatter/coord.rs | 1 - .../match_scatter/round_zero.rs | 6 +- .../server/graph_dispatch/rag_fusion/coord.rs | 4 +- .../graph_dispatch/rag_fusion/stages.rs | 6 +- .../server/graph_dispatch/shard_reads.rs | 159 +- .../server/graph_dispatch/shortest_path.rs | 4 +- .../graph_dispatch/traverse_subgraph.rs | 4 +- .../server/graph_dispatch/whole_graph.rs | 12 +- .../http/routes/query/materialized/request.rs | 1 + .../server/http/routes/query/ndjson.rs | 1 + .../server/http/routes/query_stream.rs | 2 +- .../server/http/routes/ws_rpc/execute_sql.rs | 1 + .../server/native/dispatch/graph_match.rs | 11 +- .../server/native/dispatch/raw_dispatch.rs | 2 +- .../server/native/dispatch/single_task.rs | 17 +- .../server/native/dispatch/sql_loop.rs | 8 + .../server/native/dispatch/sql_planned.rs | 4 + .../src/control/server/native/session/run.rs | 4 +- .../server/native/session/session_stream.rs | 4 +- .../pgwire/catalog/dropped_collections.rs | 1 + .../src/control/server/pgwire/handler/core.rs | 4 +- .../pgwire/handler/core/simple_query.rs | 4 +- .../server/pgwire/handler/dispatch/entry.rs | 17 +- .../pgwire/handler/dispatch/replicated.rs | 12 +- .../server/pgwire/handler/dispatch/routing.rs | 12 +- .../handler/routing/dispatch_loop/run.rs | 9 +- .../handler/routing/dispatch_loop/tracking.rs | 55 +- nodedb/src/control/server/result_stream.rs | 20 +- .../server/shared/clone_read/dispatch.rs | 2 +- .../server/shared/clone_write/probes.rs | 1 + .../control/server/shared/clone_write/util.rs | 2 +- .../ddl/neutral/collection/purge/dispatch.rs | 1 + .../ddl/neutral/kv_sorted_index/dispatch.rs | 2 +- .../ddl/neutral/maintenance/distributed.rs | 1 + .../server/shared/ddl/neutral/rate_gate.rs | 2 +- .../ddl/neutral/router/string_engine_ops.rs | 4 +- .../ddl/neutral/router/string_versioning.rs | 4 +- .../ddl/neutral/version_history/restore.rs | 44 +- .../shared/ddl/sync_dispatch/dispatch.rs | 1 + .../server/shared/ddl/user_dispatch.rs | 2 +- nodedb/src/control/server/shared/metering.rs | 2 +- .../control/server/shared/response_payload.rs | 2 +- .../server/shared/session/commit/conflict.rs | 56 - .../server/shared/session/commit/mod.rs | 4 +- .../shared/session/commit/read_split.rs | 121 ++ .../shared/session/commit/read_validation.rs | 131 +- .../server/shared/session/commit/run.rs | 68 +- .../server/shared/session/commit_calvin.rs | 7 +- .../server/shared/session/conn_scope.rs | 14 +- .../control/server/shared/session/deadline.rs | 5 +- .../server/shared/session/graph_reads.rs | 197 --- .../server/shared/session/leader_forward.rs | 2 +- .../server/shared/session/lifecycle.rs | 8 +- .../src/control/server/shared/session/mod.rs | 3 +- .../server/shared/session/own_writes.rs | 250 ++- .../shared/session/pending_shard_reads.rs | 233 +++ .../control/server/shared/session/read_set.rs | 316 ++-- .../server/shared/session/record_reads.rs | 25 +- .../server/shared/session/savepoint_ops.rs | 10 +- .../server/shared/session/served_reads.rs | 73 - .../control/server/shared/session/state.rs | 34 +- .../control/server/shared/session/store.rs | 6 +- .../shared/session/transaction/buffer.rs | 4 +- .../shared/session/transaction/lifecycle.rs | 26 +- .../shared/session/transaction/savepoints.rs | 8 +- .../server/shared/statement_exec/dispatch.rs | 8 +- .../server/shared/statement_exec/task_loop.rs | 36 +- .../predicate/txn_buffering/classify.rs | 7 +- .../server/shared/write_admission/route.rs | 2 +- .../control/server/shuffle/aggregator_hook.rs | 1 + .../control/server/shuffle/consumer_hook.rs | 1 + nodedb/src/control/server/shuffle/fanout.rs | 38 +- .../control/server/shuffle/producer_hook.rs | 34 +- .../raft_dispatch/durability_test_support.rs | 4 +- .../server/sync/raft_dispatch/propose.rs | 6 +- .../server/sync/raft_dispatch/response.rs | 2 +- nodedb/src/control/shutdown/data_plane.rs | 3 +- nodedb/src/control/state/calvin_ack_result.rs | 2 +- nodedb/src/control/state/test_core.rs | 2 +- nodedb/src/control/system_txn/scope.rs | 6 +- nodedb/src/control/system_txn/tests.rs | 2 +- nodedb/src/control/vshard_admission.rs | 25 +- nodedb/src/control/wal_replication/mod.rs | 7 +- nodedb/src/control/wal_replication/propose.rs | 16 +- .../transaction_redo/propose.rs | 3 +- .../control/wal_replication/types/aliases.rs | 28 +- .../src/control/wal_replication/types/mod.rs | 2 +- nodedb/src/control/write_resolve/propose.rs | 7 +- nodedb/src/data/executor/array_checkpoint.rs | 3 +- .../data/executor/cascade_journal_tests.rs | 1 + .../data/executor/columnar_checkpoint/load.rs | 1 + .../executor/columnar_checkpoint/write.rs | 2 +- .../executor/core_loop/event_emit_engines.rs | 4 +- .../core_loop/index_value_versions.rs | 365 +++-- .../executor/core_loop/pressure/fixtures.rs | 1 + .../executor/core_loop/priority_queues.rs | 1 + .../src/data/executor/core_loop/response.rs | 141 +- nodedb/src/data/executor/core_loop/state.rs | 4 +- nodedb/src/data/executor/core_loop/tick.rs | 3 +- .../data/executor/core_loop/write_index.rs | 1371 ----------------- .../executor/core_loop/write_index/index.rs | 544 +++++++ .../executor/core_loop/write_index/keys.rs | 48 + .../executor/core_loop/write_index/mod.rs | 35 + .../executor/core_loop/write_index/record.rs | 548 +++++++ .../core_loop/write_index/record_stamps.rs | 202 +++ .../executor/core_loop/write_index/tests.rs | 165 ++ .../core_loop/write_index/validate.rs | 389 +++++ nodedb/src/data/executor/deadline.rs | 1 + .../data/executor/dispatch/array/aggregate.rs | 1 + .../executor/dispatch/array/elementwise.rs | 1 + .../data/executor/dispatch/array/mutate.rs | 6 +- .../src/data/executor/dispatch/array/read.rs | 1 + .../executor/dispatch/array/surrogate_scan.rs | 1 + .../dispatch/bitmap/hashjoin_inline.rs | 1 + .../executor/dispatch/graph/node_labels.rs | 1 + nodedb/src/data/executor/dispatch/meta.rs | 26 +- .../src/data/executor/dispatch/timeseries.rs | 1 + .../executor/graph_label_checkpoint/mod.rs | 2 +- .../data/executor/handlers/aggregate/exec.rs | 1 + .../handlers/bulk_dml/delete_cascade.rs | 6 +- .../data/executor/handlers/bulk_dml/update.rs | 6 +- .../executor/handlers/columnar_mutation.rs | 4 +- .../columnar_read/materialize_scan.rs | 1 + .../handlers/columnar_read/scan/execute.rs | 1 + .../handlers/columnar_resolved_mutation.rs | 4 +- .../executor/handlers/columnar_truncate.rs | 2 +- .../handlers/columnar_write/insert.rs | 4 +- .../handlers/control/calvin/active_passive.rs | 4 +- .../handlers/control/calvin/shared.rs | 1 + .../handlers/control/calvin/static_stage.rs | 44 +- .../handlers/control/calvin_resolve.rs | 1 + .../data/executor/handlers/control/crdt.rs | 2 +- .../handlers/control/crdt_apply/gated.rs | 2 +- .../handlers/control/crdt_apply/local.rs | 4 +- .../executor/handlers/control/crdt_preview.rs | 16 +- .../handlers/document/apply_balance_delta.rs | 2 +- .../handlers/document/resolve/apply_row.rs | 12 +- .../handlers/document/write/batch_insert.rs | 11 +- nodedb/src/data/executor/handlers/fts_sync.rs | 4 +- .../handlers/graph_edge_write/delete.rs | 4 +- .../handlers/graph_edge_write/delete_batch.rs | 2 +- .../executor/handlers/graph_edge_write/put.rs | 4 +- .../handlers/graph_edge_write/put_batch.rs | 2 +- .../handlers/graph_edge_write/shared.rs | 9 +- .../executor/handlers/graph_node_guard.rs | 1 + .../data/executor/handlers/home_versions.rs | 59 +- .../src/data/executor/handlers/join/params.rs | 1 + .../executor/handlers/join/shuffle_join.rs | 1 + .../src/data/executor/handlers/kv/atomic.rs | 8 +- nodedb/src/data/executor/handlers/kv/batch.rs | 2 +- .../executor/handlers/kv/crud/write_basic.rs | 14 +- nodedb/src/data/executor/handlers/kv/field.rs | 2 +- .../executor/handlers/kv/predicate/apply.rs | 2 +- .../executor/handlers/kv/resolve/apply.rs | 10 +- nodedb/src/data/executor/handlers/kv/ttl.rs | 4 +- .../executor/handlers/point/apply_put/core.rs | 1 + .../handlers/point/apply_put/index.rs | 1 + .../data/executor/handlers/point/delete.rs | 6 +- .../data/executor/handlers/point/insert.rs | 6 +- .../src/data/executor/handlers/point/put.rs | 6 +- .../executor/handlers/point/update/exec.rs | 5 + .../executor/handlers/point/update/persist.rs | 9 +- .../handlers/query_collection_size.rs | 2 +- .../handlers/snapshot/capture_arrays.rs | 8 +- .../handlers/snapshot/install_arrays.rs | 4 +- .../executor/handlers/spatial/rtree_scan.rs | 1 + .../data/executor/handlers/spatial_sync.rs | 4 +- .../executor/handlers/timeseries/ingest.rs | 2 +- .../handlers/timeseries/ingest_dispatch.rs | 4 +- .../handlers/timeseries/ingest_resolved.rs | 2 +- .../data/executor/handlers/timeseries/scan.rs | 2 +- .../executor/handlers/timeseries/truncate.rs | 2 +- .../data/executor/handlers/timeseries_wal.rs | 50 +- .../redo_apply/calvin_install_tests.rs | 19 +- .../handlers/transaction/redo_apply/entry.rs | 11 +- .../redo_apply/install_refusal_tests.rs | 10 +- .../handlers/transaction/redo_apply/state.rs | 9 +- .../handlers/transaction/resolve/entry.rs | 1 + .../transaction/stage_write/stage_graph.rs | 1 + .../transaction/stage_write/stage_kv.rs | 1 + .../stage_write/stage_kv_predicate.rs | 1 + .../stage_write/stage_timeseries.rs | 1 + .../transaction/stage_write/stage_vector.rs | 1 + .../handlers/transaction/undo/apply.rs | 1 + .../handlers/transaction/write_version.rs | 83 +- .../handlers/transaction/write_version_kv.rs | 21 +- .../transaction/write_version_parity_tests.rs | 12 +- nodedb/src/data/executor/handlers/truncate.rs | 3 + .../handlers/update_from_join_write.rs | 4 +- .../executor/handlers/upsert/exec/insert.rs | 16 + .../handlers/upsert/exec/overwrite.rs | 16 + nodedb/src/data/executor/handlers/vector.rs | 20 +- .../handlers/vector_direct_resolve/apply.rs | 1 + .../executor/handlers/vector_direct_row.rs | 2 +- .../data/executor/handlers/vector_multi.rs | 20 +- .../data/executor/handlers/vector_sparse.rs | 18 +- .../data/executor/handlers/vector_upsert.rs | 16 +- .../data/executor/handlers/vector_write.rs | 20 +- nodedb/src/data/executor/replay_task.rs | 1 + nodedb/src/data/executor/task.rs | 3 +- .../executor/timeseries_checkpoint/flush.rs | 3 +- .../executor/timeseries_checkpoint/mod.rs | 2 +- nodedb/src/data/executor/wal_replay/crdt.rs | 26 +- nodedb/src/data/executor/wal_replay/kv.rs | 10 +- nodedb/src/data/executor/wal_replay/kv_put.rs | 6 +- nodedb/src/data/executor/wal_replay_all.rs | 159 +- .../executor/wal_replay_columnar_image.rs | 2 +- nodedb/src/data/executor/wal_replay_fts.rs | 1 + .../src/data/executor/wal_replay_kv_atomic.rs | 6 +- .../src/data/executor/wal_replay_kv_expiry.rs | 4 +- .../src/data/executor/wal_replay_kv_field.rs | 2 +- .../src/data/executor/wal_replay_kv_incr.rs | 2 +- .../executor/wal_replay_kv_insert_conflict.rs | 2 +- .../data/executor/wal_replay_kv_predicate.rs | 4 +- .../data/executor/wal_replay_kv_transfer.rs | 8 +- nodedb/src/data/executor/wal_replay_kv_ttl.rs | 1 + .../data/executor/wal_replay_redo_document.rs | 35 +- .../data/executor/wal_replay_redo_graph.rs | 14 +- .../executor/wal_replay_redo_graph_cut.rs | 1 + .../executor/wal_replay_row_image_tests.rs | 1 + .../src/data/executor/wal_replay_spatial.rs | 1 + .../data/executor/wal_replay_vector_task.rs | 1 + .../data/executor/write_set_crash_tests.rs | 1 + nodedb/src/data/runtime/event_loop.rs | 2 +- nodedb/src/engine/array/store/catalog/scan.rs | 19 +- nodedb/src/types/mod.rs | 2 + nodedb/src/types/read_versions.rs | 115 ++ nodedb/tests/crash_harness/vshards.rs | 19 +- nodedb/tests/crash_replay_stamp/engines.rs | 13 +- .../cases/calvin_determinism_contract.rs | 1 + .../inproc/cases/calvin_executor_apply.rs | 1 + .../cases/calvin_two_phase_apply/phantom.rs | 15 +- .../cases/calvin_two_phase_apply/read_set.rs | 50 +- .../cases/calvin_two_phase_apply/support.rs | 15 + .../cases/cross_engine_bitmap_currency.rs | 1 + .../cross_engine_three_way_fts_vector_doc.rs | 1 + .../inproc/cases/executor_tests/helpers.rs | 1 + .../cases/request_tracker_backpressure.rs | 2 +- .../tests/inproc/cases/startup_gate_pgwire.rs | 2 +- .../inproc/cases/surrogate_round_trip.rs | 1 + .../cases/wal_catchup/startup_replay.rs | 1 + 386 files changed, 6067 insertions(+), 3920 deletions(-) create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_cluster_array_read_set.rs create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_read_version_leader_move.rs create mode 100644 nodedb-types/src/write_version.rs create mode 100644 nodedb/src/control/cluster/snapshot_install/version_floor.rs delete mode 100644 nodedb/src/control/server/shared/session/commit/conflict.rs create mode 100644 nodedb/src/control/server/shared/session/commit/read_split.rs delete mode 100644 nodedb/src/control/server/shared/session/graph_reads.rs create mode 100644 nodedb/src/control/server/shared/session/pending_shard_reads.rs delete mode 100644 nodedb/src/control/server/shared/session/served_reads.rs delete mode 100644 nodedb/src/data/executor/core_loop/write_index.rs create mode 100644 nodedb/src/data/executor/core_loop/write_index/index.rs create mode 100644 nodedb/src/data/executor/core_loop/write_index/keys.rs create mode 100644 nodedb/src/data/executor/core_loop/write_index/mod.rs create mode 100644 nodedb/src/data/executor/core_loop/write_index/record.rs create mode 100644 nodedb/src/data/executor/core_loop/write_index/record_stamps.rs create mode 100644 nodedb/src/data/executor/core_loop/write_index/tests.rs create mode 100644 nodedb/src/data/executor/core_loop/write_index/validate.rs create mode 100644 nodedb/src/types/read_versions.rs diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_cluster_array_read_set.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_cluster_array_read_set.rs new file mode 100644 index 000000000..a1415884f --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_cluster_array_read_set.rs @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A transaction's cluster array read joins its read set. +//! +//! A 3-node cluster holds the array `grid`, whose cells spread over the tile +//! vShards of several nodes, and the keyed collection `log`. A transaction +//! reads an `ARRAY_SLICE` that spans the tiles of several shards, then +//! writes `log`. Each shard leg reports its vShard's write version, even +//! when it matched no cell, and COMMIT validates the read on every covered +//! vShard. +//! +//! - No write to `grid` after the read: the COMMIT succeeds. +//! - A cell written to a covered tile after the read moves that vShard's +//! version past the read's, so the COMMIT aborts with SQLSTATE 40001 and +//! the transaction's write is not visible. + +use super::calvin_multishard_fixture::{Fixture, data_rows, keyed_ddl, row_count}; +use super::calvin_replica_content::{run_retrying, strict_session}; +use crate::common::cluster_harness::shared_steps::db_detail; + +/// SQLSTATE of a transaction aborted by a conflict. +const SERIALIZATION_FAILURE: &str = "40001"; + +/// The slice the transaction reads: three tiles, one per `chr`. +const SLICE: &str = + "SELECT * FROM ARRAY_SLICE('grid', '{chr: [0, 2], pos: [0, 99]}', ['qual'], 100)"; + +/// A 3-node cluster holding the array `grid`, seeded with three cells per +/// tile on `chr` 0 to 2, and the keyed collection `log`. +async fn seeded(log: &str) -> Fixture { + let fx = Fixture::spawn(&[keyed_ddl(log)]).await; + fx.wait_group_mounted(log).await; + fx.cluster + .exec_ddl_on_any_leader( + "CREATE ARRAY grid \ + DIMS (chr INT64 [0..9], pos INT64 [0..99]) \ + ATTRS (qual FLOAT64) \ + TILE_EXTENTS (1, 100) \ + CELL_ORDER HILBERT", + ) + .await + .unwrap_or_else(|e| panic!("CREATE ARRAY grid: {e}")); + run_retrying( + &fx.cluster.nodes[0].client, + "seed the array", + "INSERT INTO ARRAY grid \ + COORDS (0, 10) VALUES (1.0), \ + COORDS (0, 20) VALUES (2.0), \ + COORDS (1, 10) VALUES (10.0), \ + COORDS (1, 20) VALUES (20.0), \ + COORDS (2, 10) VALUES (100.0), \ + COORDS (2, 20) VALUES (200.0)", + ) + .await; + fx.converge().await; + fx +} + +/// Open a transaction on the coordinator that reads the slice and buffers a +/// write to `log`, and return its session. +async fn read_slice_then_buffer_write(fx: &Fixture, log: &str) -> tokio_postgres::Client { + let session = strict_session(fx.coordinator()).await; + session + .simple_query("BEGIN") + .await + .unwrap_or_else(|e| panic!("BEGIN: {}", db_detail(&e))); + let rows = session + .simple_query(SLICE) + .await + .unwrap_or_else(|e| panic!("in-transaction slice: {}", db_detail(&e))); + assert_eq!(data_rows(&rows), 6, "the slice reads every seeded cell"); + session + .simple_query(&format!("INSERT INTO {log} (id, v) VALUES ('w1', 'txn')")) + .await + .unwrap_or_else(|e| panic!("buffer the write: {}", db_detail(&e))); + session +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_current_cluster_array_read_commits() { + let log = "car_ok_log"; + let fx = seeded(log).await; + let session = read_slice_then_buffer_write(&fx, log).await; + + session.simple_query("COMMIT").await.unwrap_or_else(|e| { + panic!( + "no write reached the slice's tiles, so the commit succeeds: {}", + db_detail(&e) + ) + }); + fx.converge().await; + assert_eq!( + row_count( + &fx.coordinator().client, + &format!("SELECT id FROM {log} WHERE id = 'w1'") + ) + .await, + 1, + "the committed transaction's write is visible" + ); + + fx.cluster.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_write_to_a_covered_tile_aborts_the_cluster_array_read() { + let log = "car_stale_log"; + let fx = seeded(log).await; + let session = read_slice_then_buffer_write(&fx, log).await; + + let writer = (fx.coordinator + 1) % fx.cluster.nodes.len(); + run_retrying( + &fx.cluster.nodes[writer].client, + "a conflicting write to a covered tile", + "INSERT INTO ARRAY grid COORDS (1, 50) VALUES (5.0)", + ) + .await; + fx.converge().await; + + let error = session + .simple_query("COMMIT") + .await + .expect_err("a cell written to a covered tile after the read makes it stale"); + assert_eq!( + error.as_db_error().map(|db| db.code().code()), + Some(SERIALIZATION_FAILURE), + "the stale array read aborts the commit with a serialization failure: {}", + db_detail(&error) + ); + fx.converge().await; + assert_eq!( + row_count( + &fx.coordinator().client, + &format!("SELECT id FROM {log} WHERE id = 'w1'") + ) + .await, + 0, + "the aborted transaction's write is not visible" + ); + + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_read_version_leader_move.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_read_version_leader_move.rs new file mode 100644 index 000000000..41fb4f3d1 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_read_version_leader_move.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A transaction's read validates at whichever replica leads its vShard at +//! COMMIT. +//! +//! A transaction reads a row of `reads`, then writes `writes` on another +//! vShard, so its COMMIT runs through Calvin with the read's vShard as a +//! read-only participant. The read's version is the data-group log position +//! of the last write to `reads`, the same on every replica. +//! +//! The read runs while a non-preferred replica leads the read's data group. +//! Before the COMMIT, leadership moves back to the preferred replica. The +//! leader balance never moves a group away from its preferred leader, so that +//! replica, never the one that served the read, validates it. +//! +//! - No write to `reads` after the read: the validating leader finds the +//! read current, and the COMMIT succeeds. +//! - A write to the read row committed after the read: its log position is +//! above the read's version, so the validating leader aborts the COMMIT +//! with SQLSTATE 40001. + +use std::time::Duration; + +use super::calvin_multishard_fixture::{Fixture, data_rows, keyed_ddl, row_count}; +use super::calvin_replica_content::{names_on_distinct_vshards, run_retrying, strict_session}; +use crate::common::cluster_harness::GroupLeader; +use crate::common::cluster_harness::shared_steps::{db_detail, group_members, group_of}; + +/// SQLSTATE of a transaction aborted by a conflict. +const SERIALIZATION_FAILURE: &str = "40001"; + +/// How long a group gets to show a leader. +const LEADER_WAIT: Duration = Duration::from_secs(30); + +/// Attempts at a read served by the non-preferred leader. The leader balance +/// can move the group back between the move and the read. The attempt then +/// rolls back and starts again. +const READ_ATTEMPTS: usize = 10; + +/// A 3-node cluster holding `reads` and `writes` on distinct vShards, with +/// row `r1` committed in `reads` and replicated everywhere. +async fn seeded(prefixes: [&str; 2]) -> (Fixture, String, String) { + let [reads, writes] = names_on_distinct_vshards(prefixes); + let fx = Fixture::spawn(&[keyed_ddl(&reads), keyed_ddl(&writes)]).await; + for collection in [&reads, &writes] { + fx.wait_group_mounted(collection).await; + } + run_retrying( + &fx.cluster.nodes[0].client, + "seed the read row", + &format!("INSERT INTO {reads} (id, v) VALUES ('r1', 'seed')"), + ) + .await; + fx.converge().await; + (fx, reads, writes) +} + +/// The leaders a case moves `reads`' data group between. +struct Leaders { + group: u64, + /// The preferred leader, which validates the read at COMMIT. + validating: u64, + /// Another replica, which serves the read. + serving: u64, +} + +/// Settle every group on its preferred leader and pick the two leaders of +/// `collection`'s data group. +async fn leaders_of(fx: &Fixture, collection: &str) -> Leaders { + fx.cluster.wait_for_preferred_leaders(LEADER_WAIT).await; + let group = group_of(&fx.cluster.nodes[0], collection); + let validating = fx + .cluster + .wait_for_group_leader(group, GroupLeader::Any, LEADER_WAIT) + .await; + let serving = group_members(&fx.cluster.nodes[0], group) + .into_iter() + .find(|member| *member != validating) + .unwrap_or_else(|| panic!("group {group} has a replica besides its leader {validating}")); + Leaders { + group, + validating, + serving, + } +} + +/// Open a transaction on the coordinator whose read of `r1` from `reads` +/// the serving leader answered, and buffer a write to `writes`. Return its +/// session. +async fn read_on_serving_leader( + fx: &Fixture, + leaders: &Leaders, + reads: &str, + writes: &str, +) -> tokio_postgres::Client { + let session = strict_session(fx.coordinator()).await; + for _ in 0..READ_ATTEMPTS { + fx.cluster + .transfer_leadership(leaders.group, leaders.serving) + .await; + session + .simple_query("BEGIN") + .await + .unwrap_or_else(|e| panic!("BEGIN: {}", db_detail(&e))); + let rows = session + .simple_query(&format!("SELECT v FROM {reads} WHERE id = 'r1'")) + .await + .unwrap_or_else(|e| panic!("in-transaction read: {}", db_detail(&e))); + assert_eq!(data_rows(&rows), 1, "the transaction reads the seeded row"); + if fx.cluster.group_leader(leaders.group) != Some(leaders.serving) { + session + .simple_query("ROLLBACK") + .await + .unwrap_or_else(|e| panic!("ROLLBACK: {}", db_detail(&e))); + continue; + } + session + .simple_query(&format!( + "INSERT INTO {writes} (id, v) VALUES ('w1', 'txn')" + )) + .await + .unwrap_or_else(|e| panic!("buffer the write: {}", db_detail(&e))); + return session; + } + panic!( + "node {} did not hold the leadership of group {} across a read in {READ_ATTEMPTS} attempts", + leaders.serving, leaders.group + ); +} + +/// Move the leadership back to the preferred leader, which then validates +/// the read at COMMIT. +async fn hand_back(fx: &Fixture, leaders: &Leaders) { + fx.cluster + .transfer_leadership(leaders.group, leaders.validating) + .await; + fx.cluster + .wait_for_group_leader( + leaders.group, + GroupLeader::Node(leaders.validating), + LEADER_WAIT, + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_current_read_commits_after_its_leader_moves() { + let (fx, reads, writes) = seeded(["lvm_ok_reads", "lvm_ok_writes"]).await; + let leaders = leaders_of(&fx, &reads).await; + let session = read_on_serving_leader(&fx, &leaders, &reads, &writes).await; + hand_back(&fx, &leaders).await; + + session.simple_query("COMMIT").await.unwrap_or_else(|e| { + panic!( + "a read the serving leader answered is current at the validating \ + leader, so the commit succeeds: {}", + db_detail(&e) + ) + }); + fx.converge().await; + assert_eq!( + row_count( + &fx.coordinator().client, + &format!("SELECT id FROM {writes} WHERE id = 'w1'") + ) + .await, + 1, + "the committed transaction's write is visible" + ); + + fx.cluster.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_write_after_the_read_aborts_the_commit_after_its_leader_moves() { + let (fx, reads, writes) = seeded(["lvm_stale_reads", "lvm_stale_writes"]).await; + let leaders = leaders_of(&fx, &reads).await; + let session = read_on_serving_leader(&fx, &leaders, &reads, &writes).await; + hand_back(&fx, &leaders).await; + let writer = (fx.coordinator + 1) % fx.cluster.nodes.len(); + run_retrying( + &fx.cluster.nodes[writer].client, + "a conflicting write to the read row", + &format!("UPDATE {reads} SET v = 'moved' WHERE id = 'r1'"), + ) + .await; + fx.converge().await; + + let error = session + .simple_query("COMMIT") + .await + .expect_err("a write committed after the read makes the read stale"); + assert_eq!( + error.as_db_error().map(|db| db.code().code()), + Some(SERIALIZATION_FAILURE), + "the stale read aborts the commit with a serialization failure: {}", + db_detail(&error) + ); + fx.converge().await; + assert_eq!( + row_count( + &fx.coordinator().client, + &format!("SELECT id FROM {writes} WHERE id = 'w1'") + ) + .await, + 0, + "the aborted transaction's write is not visible" + ); + + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/cross_node_read_occ_abort.rs b/nodedb-cluster-tests/tests/common_suite/cases/cross_node_read_occ_abort.rs index 109d52bb7..f87bd2781 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/cross_node_read_occ_abort.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/cross_node_read_occ_abort.rs @@ -8,11 +8,10 @@ //! An interactive transaction can WRITE several collections and READ another. To //! exercise the cross-shard OCC read-validation path, the transaction must go //! through the multi-participant Calvin barrier (`run_commit_calvin`) rather than -//! the single-shard local-WAL `si_conflict_abort` fast path. That barrier is the -//! only place that revalidates a read slice on its owning vShard using the real -//! per-shard `read_lsn`, by dispatching a validate-only task to every read-only -//! participant. If a read-only participant is never validated, a stale read -//! commits silently — a non-serializable execution. +//! the single-shard path. The barrier revalidates a read slice on its owning +//! vShard at the read's real version, by dispatching a validate-only task to +//! every read-only participant. If a read-only participant is never validated, +//! a stale read commits silently — a non-serializable execution. //! //! ## Why a single node reproduces it //! @@ -230,7 +229,7 @@ async fn cross_shard_read_occ_abort_on_stale_read() { // writes, leaving the transaction OPEN (no COMMIT). The two INSERTs buffer on // the coordinator, so at COMMIT the write set spans two vShards → participant // floor >= 2 → the multi-participant Calvin barrier; the SELECT registers - // bread's vShard as a read-only participant with a captured read_lsn. + // bread's vShard as a read-only participant with a captured read version. let block = format!( "BEGIN; \ SELECT * FROM {bread}; \ @@ -284,8 +283,8 @@ async fn cross_shard_read_occ_abort_on_stale_read() { pg_detail(&err) ); - // Discriminator: `si_conflict_abort`'s single-shard fast path raises this same - // SQLSTATE without ever admitting the batch to a sequencer epoch, so the 40001 + // Discriminator: the single-shard commit check raises this same SQLSTATE + // without ever admitting the batch to a sequencer epoch, so the 40001 // assertion above alone cannot tell "aborted by the multi-participant Calvin // barrier revalidating bread's read-only-participant slice" apart from "aborted // for some unrelated single-shard reason". Only the barrier path admits the @@ -318,7 +317,7 @@ async fn cross_shard_read_occ_abort_on_stale_read() { /// The precision control: the SAME cross-shard shape (read `bread`, write /// `w1`/`w2`) with NO concurrent write to `bread` must COMMIT SUCCESSFULLY. The -/// barrier validates `bread`'s read at its real `read_lsn` and finds it still +/// barrier validates `bread`'s read at its real read version and finds it still /// current — proving the abort above is caused by the genuine version advance, /// not a blanket cross-shard abort. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] diff --git a/nodedb-cluster-tests/tests/common_suite/cases/gather_join_in_txn_occ.rs b/nodedb-cluster-tests/tests/common_suite/cases/gather_join_in_txn_occ.rs index 560499088..6c9c4c574 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/gather_join_in_txn_occ.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/gather_join_in_txn_occ.rs @@ -11,7 +11,7 @@ //! where the left side is scanned locally, and GATHERS the RIGHT (build) //! collection across all vShards (`gather_join_build_side`) to inline it as a //! `ProviderScan`. Each of those two gathers observes its own collection's -//! `coll_write_lsn` at read time. +//! write version at read time. //! //! The GATHER path must not have a serializability HOLE where only the plan's //! collapsed left collection reaches the read-set (`extract_collection` of an @@ -28,7 +28,7 @@ //! * `commits_when_neither_side_concurrently_written` — neither join side is //! written during the txn, so both recorded real versions still validate and //! COMMIT must SUCCEED (proves the captures carry the sound per-collection -//! `coll_write_lsn`, not an inflated global watermark that would over-abort). +//! write version, not an inflated global watermark that would over-abort). //! * `occ_aborts_on_stale_probe_read` — a confirmed-visible concurrent write to //! the LEFT/probe collection advances it past the captured probe version, so //! COMMIT must abort with 40001. @@ -183,7 +183,7 @@ async fn spawn_node_with_collections() -> ( .await; // Seed left and right with three matching join keys so the join returns real - // rows AND each side's committed `coll_write_lsn` is non-zero (gives the + // rows AND each side's committed write version is non-zero (gives the // stale-read cases a real baseline version to advance past, and the // commits-clean case a non-zero version that must still validate). node.client @@ -267,7 +267,7 @@ async fn begin_join_read_and_buffer_writes( /// Neither join side is concurrently written, so both recorded real read versions /// still validate at the barrier and COMMIT must SUCCEED. Proves the captured -/// read-versions are the sound per-collection `coll_write_lsn` — an inflated +/// read-versions are the sound per-collection write versions — an inflated /// global watermark would over-abort this clean commit. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn gather_join_commits_when_neither_side_concurrently_written() { diff --git a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs index fdd6b81f5..5a1b58784 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs @@ -13,6 +13,7 @@ mod backup_schedule_failover; mod bitemporal_array_cluster; mod calvin_backup_cut_order; mod calvin_cdc_net_kinds; +mod calvin_cluster_array_read_set; mod calvin_cluster_pgwire_e2e; mod calvin_duplicate_redo_copy; mod calvin_joiner_after_sequencer_compaction; @@ -28,6 +29,7 @@ mod calvin_old_term_redo_copy; mod calvin_ollp_cross_node; mod calvin_ollp_pk_delete; mod calvin_ollp_update; +mod calvin_read_version_leader_move; mod calvin_remote_participant_report; mod calvin_replica_content; mod calvin_replica_convergence; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/native_gather_join_in_txn_occ.rs b/nodedb-cluster-tests/tests/common_suite/cases/native_gather_join_in_txn_occ.rs index 047bd61d7..6536f34a3 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/native_gather_join_in_txn_occ.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/native_gather_join_in_txn_occ.rs @@ -15,13 +15,13 @@ //! //! Cases bounding the fixed behavior: //! * `commits_when_neither_side_concurrently_written` — clean commit MUST SUCCEED -//! (captures carry the sound `coll_write_lsn`, not an over-aborting watermark). +//! (captures carry the sound collection write version, not an over-aborting watermark). //! * `occ_aborts_on_stale_probe_read` — a stale LEFT/probe read MUST abort. //! * `occ_aborts_on_stale_build_read` — a stale RIGHT/build read (pgwire rival) //! MUST abort: the native build-side capture hole is closed on the Gather path. //! * `occ_aborts_on_native_writer_stale_read` — the build-side rival is a NATIVE //! autocommit INSERT. Pre-fix the gateway leader-local autocommit path never -//! proposed a Raft entry nor bumped `coll_write_lsn`, so the read never went +//! proposed a Raft entry nor bumped the collection write version, so the read never went //! stale and COMMIT silently succeeded; routing native writes through Raft //! closes that lost-update cell. //! @@ -177,7 +177,7 @@ async fn spawn_node_with_collections() -> ( .await; // Seed left and right with three matching join keys so the join returns real - // rows AND each side's committed `coll_write_lsn` is non-zero (gives the + // rows AND each side's committed write version is non-zero (gives the // stale-read cases a real baseline version to advance past, and the // commits-clean case a non-zero version that must still validate). node.client @@ -300,7 +300,7 @@ async fn concurrent_write_and_confirm(node: &TestClusterNode, coll: &str) { /// NATIVE concurrent writer: the rival autocommit INSERT rides a SEPARATE native /// client, exercising the gateway leader-local autocommit-write path that must now -/// PROPOSE through Raft and bump `coll_write_lsn`. Confirmed visible before COMMIT. +/// PROPOSE through Raft and bump the collection write version. Confirmed visible before COMMIT. async fn native_concurrent_write_and_confirm(node: &TestClusterNode, coll: &str) { let writer = pinned_native_client(node); writer @@ -314,7 +314,7 @@ async fn native_concurrent_write_and_confirm(node: &TestClusterNode, coll: &str) /// Neither join side is concurrently written, so both recorded real read versions /// still validate at the barrier and COMMIT must SUCCEED. Proves the captured -/// read-versions are the sound per-collection `coll_write_lsn` — an inflated +/// read-versions are the sound per-collection write versions — an inflated /// global watermark would over-abort this clean commit. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn native_gather_join_commits_when_neither_side_concurrently_written() { @@ -464,7 +464,7 @@ async fn native_gather_join_occ_aborts_on_stale_build_read() { /// rival autocommit INSERT rides a SEPARATE native client, driving the gateway's /// leader-local autocommit-write path. Pre-fix that path applied the row via a /// leader-local SPSC dispatch that never proposed a Raft entry nor bumped the build -/// collection's `coll_write_lsn`, so the captured build version never went stale +/// collection's write version, so the captured build version never went stale /// and the COMMIT SILENTLY succeeded (undetected lost-update). Routing native /// autocommit writes through Raft advances the build side, so COMMIT must abort. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] diff --git a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_aggregate_in_txn_occ.rs b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_aggregate_in_txn_occ.rs index 5d5926494..0c4e37698 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_aggregate_in_txn_occ.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_aggregate_in_txn_occ.rs @@ -8,16 +8,16 @@ //! An interactive transaction can WRITE several collections and READ another via //! a distributed GROUP BY that the planner lowers to `Exchange{ShuffleAggregate}`. //! The coordinator fans partial-state PRODUCERs to the source collection's -//! owner(s); each producer scans the collection and — with the producer-side -//! read-version fix — reports the scanned collection's `coll_write_lsn` at read -//! time back on its `ShuffleProduceResponse`. The coordinator max-folds those and -//! records the aggregate's read-set entry with that version. +//! owner(s); each producer scans the collection and reports the scanned +//! collection's write version at read time back on its +//! `ShuffleProduceResponse`. The coordinator max-folds those and records the +//! aggregate's read-set entry with that version. //! -//! The shuffle-aggregate resolver must not record `Lsn::ZERO` as the +//! The shuffle-aggregate resolver must not record `WriteVersion::ZERO` as the //! read version. At the cross-shard COMMIT barrier the read-only participant is -//! revalidated with `coll_write_lsn(coll) <= read_version`; with `read_version = -//! 0` and any prior write to the collection (`coll_write_lsn > 0`) the comparison -//! is `>0 <= 0` → false → a SPURIOUS serialization abort (SQLSTATE 40001) for a +//! revalidated with `write_version(coll) <= read_version`. With a zero read +//! version and any prior write to the collection, the write version is above +//! zero, so the comparison is false → a SPURIOUS serialization abort (SQLSTATE 40001) for a //! read that was never concurrently written. This suite bounds the behavior from //! BOTH sides: //! @@ -148,7 +148,7 @@ async fn spawn_node_with_collections() .await; // Seed metrics with committed low-cardinality GROUP BY rows so the aggregate - // read observes real data AND metrics' committed `coll_write_lsn` is non-zero + // read observes real data AND metrics' committed write version is non-zero // (the exact precondition that made the pre-fix ZERO read-version abort). node.client .simple_query(&format!( @@ -197,8 +197,8 @@ async fn spawn_node_with_collections() /// version still validates at the barrier and COMMIT must SUCCEED. This is the /// direction the producer-side ZERO→real read-version fix unblocks: before it, /// the shuffle-aggregate recorded read version `0`, and the barrier's -/// `coll_write_lsn(metrics) <= 0` check (metrics has committed writes, so -/// `coll_write_lsn > 0`) spuriously aborted the COMMIT with 40001. +/// `write_version(metrics) <= 0` check (metrics has committed writes, so its +/// write version is above zero) spuriously aborted the COMMIT with 40001. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn shuffle_aggregate_read_commits_when_not_concurrently_written() { let (node, _data_dir, metrics, w1, w2) = spawn_node_with_collections().await; @@ -273,7 +273,7 @@ async fn shuffle_aggregate_read_commits_when_not_concurrently_written() { /// The precision control: the SAME shape, but a confirmed-visible concurrent write /// advances `metrics` past the captured aggregate read version. COMMIT must STILL /// abort with SQLSTATE 40001 — the barrier revalidates metrics' read-only slice -/// at its real `read_lsn` and finds it stale. Guards the fix against becoming a +/// at its real read version and finds it stale. Guards the fix against becoming a /// blanket "always valid". #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn shuffle_aggregate_read_occ_aborts_on_stale_read() { diff --git a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_in_txn_occ.rs b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_in_txn_occ.rs index 3c5c75d43..1aad293fe 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_in_txn_occ.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/shuffle_join_in_txn_occ.rs @@ -10,7 +10,7 @@ //! a `HashJoin`. The coordinator fans PRODUCERs to EACH side's owner: the probe //! (left) side scans `left_collection`, the build (right) side scans //! `right_collection`. Each producer reports the scanned collection's -//! `coll_write_lsn` at read time on its `ShuffleProduceResponse`; the coordinator +//! write version at read time on its `ShuffleProduceResponse`; the coordinator //! records ONE read-set entry PER SIDE, each stamped with that side's real //! max-folded read version. //! @@ -159,7 +159,7 @@ async fn spawn_node_with_collections() -> ( .await; // Seed left and right with three matching join keys so the join returns real - // rows AND each side's committed `coll_write_lsn` is non-zero (the exact + // rows AND each side's committed write version is non-zero (the exact // precondition that made the pre-fix ZERO probe read-version spuriously abort, // and gives the stale-read cases a real baseline version to advance past). node.client @@ -239,8 +239,8 @@ async fn begin_join_read_and_buffer_writes( /// Neither join side is concurrently written, so both recorded real read versions /// still validate at the barrier and COMMIT must SUCCEED. This is the direction /// the fix unblocks: before it, the probe (left) side recorded read version `0` -/// and the barrier's `coll_write_lsn(left) <= 0` check (left has committed writes, -/// so `coll_write_lsn > 0`) spuriously aborted the COMMIT with 40001. +/// and the barrier's `write_version(left) <= 0` check (left has committed writes, +/// so its write version is above zero) spuriously aborted the COMMIT with 40001. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn shuffle_join_commits_when_neither_side_concurrently_written() { let (node, _data_dir, left, right, w1, w2) = spawn_node_with_collections().await; diff --git a/nodedb-cluster-tests/tests/misc_suite/cases/transport_security.rs b/nodedb-cluster-tests/tests/misc_suite/cases/transport_security.rs index 95910aa4c..8ae10a2e1 100644 --- a/nodedb-cluster-tests/tests/misc_suite/cases/transport_security.rs +++ b/nodedb-cluster-tests/tests/misc_suite/cases/transport_security.rs @@ -118,7 +118,7 @@ impl RaftRpcHandler for EchoHandler { ) -> nodedb_cluster::rpc_codec::ShuffleProduceResponse { nodedb_cluster::rpc_codec::ShuffleProduceResponse { error: None, - read_version_lsn: 0, + read_versions: Vec::new(), } } diff --git a/nodedb-cluster-tests/tests/misc_suite/cases/wire_version_handshake.rs b/nodedb-cluster-tests/tests/misc_suite/cases/wire_version_handshake.rs index 22efa1c23..ed7e01bf5 100644 --- a/nodedb-cluster-tests/tests/misc_suite/cases/wire_version_handshake.rs +++ b/nodedb-cluster-tests/tests/misc_suite/cases/wire_version_handshake.rs @@ -84,7 +84,7 @@ impl RaftRpcHandler for EchoHandler { ) -> nodedb_cluster::rpc_codec::ShuffleProduceResponse { nodedb_cluster::rpc_codec::ShuffleProduceResponse { error: None, - read_version_lsn: 0, + read_versions: Vec::new(), } } @@ -224,7 +224,7 @@ impl RaftRpcHandler for SentinelHandler { code: 0, message: "sentinel: unexpected produce dispatch".into(), }), - read_version_lsn: 0, + read_versions: Vec::new(), } } diff --git a/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/calvin_read_failover_cross_node.rs b/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/calvin_read_failover_cross_node.rs index 7686335ea..7c997a033 100644 --- a/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/calvin_read_failover_cross_node.rs +++ b/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/calvin_read_failover_cross_node.rs @@ -1,15 +1,19 @@ // SPDX-License-Identifier: BUSL-1.1 -//! A read-write transaction's read is never validated against another node's -//! WAL. +//! A read made before a leader change validates at the new leader. //! //! A transaction reads a row of a collection whose home group leader is `A`, -//! so `A` serves the read and its version is a position in `A`'s WAL. `A` is -//! then cut off from both other nodes, they elect `B`, and a write to the read -//! row commits at `B`. The partition heals. The transaction also writes, so -//! its COMMIT goes through Calvin, and `B` validates the read. `B`'s versions -//! do not compare with `A`'s, so the commit must abort with a retryable -//! serialization error. +//! so `A` serves the read. Its version is the data-group log position of the +//! last write to the collection, the same on every replica. `A` is then cut +//! off from both other nodes, they elect `B`, and a write to the read row +//! commits at `B` at a later log position. The partition heals. The +//! transaction also writes, so its COMMIT goes through Calvin, and `B` +//! validates the read. `B`'s version of the read row is above the read's, +//! so the commit must abort with a retryable serialization error. +//! +//! The control case runs the same leader change with no write to the read +//! row. Its commit must succeed, so the leader change alone never aborts a +//! transaction, and the abort above comes from the write. use std::time::Duration; @@ -55,8 +59,12 @@ fn link(cluster: &TestCluster, a: usize, b: usize, severed: bool) { } } -#[tokio::test(flavor = "multi_thread", worker_threads = 6)] -async fn a_read_served_by_a_deposed_leader_aborts_the_calvin_commit() { +/// Read the seeded row in a transaction served by the old leader, move the +/// group's leadership by a partition, write the read row at the new leader +/// when `conflict` is set, heal, and return the transaction's COMMIT result. +async fn commit_after_a_leader_change( + conflict: bool, +) -> (TestCluster, nodedb_types::error::NodeDbResult<()>) { let cluster = TestCluster::spawn_three().await.expect("3-node cluster"); for coll in [DOCS, LOG] { cluster @@ -140,34 +148,48 @@ async fn a_read_served_by_a_deposed_leader_aborts_the_calvin_commit() { }, ) .await; - let conflicting = format!("UPDATE {DOCS} SET value = '2' WHERE id = 'a'"); - wait_for_async( - "the conflicting write commits at the new leader", - Duration::from_secs(30), - Duration::from_millis(200), - || async { - cluster.nodes[writer] - .client - .simple_query(&conflicting) - .await - .is_ok() - }, - ) - .await; + if conflict { + let conflicting = format!("UPDATE {DOCS} SET value = '2' WHERE id = 'a'"); + wait_for_async( + "the conflicting write commits at the new leader", + Duration::from_secs(30), + Duration::from_millis(200), + || async { + cluster.nodes[writer] + .client + .simple_query(&conflicting) + .await + .is_ok() + }, + ) + .await; + } link(&cluster, old, reader, false); link(&cluster, old, writer, false); cluster .wait_for_full_apply_convergence(Duration::from_secs(30)) .await; - let err = driver - .commit() - .await - .expect_err("a read the deposed leader served must not validate at the new leader"); + let committed = driver.commit().await; + (cluster, committed) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_write_after_the_read_aborts_the_commit_across_a_leader_change() { + let (cluster, committed) = commit_after_a_leader_change(true).await; + let err = committed + .expect_err("a write committed after the read must abort the commit at the new leader"); assert!( err.message().contains(SERIALIZATION_ABORT), "expected a retryable serialization abort, got: {err}" ); + cluster.shutdown().await; +} +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_current_read_commits_across_a_leader_change() { + let (cluster, committed) = commit_after_a_leader_change(false).await; + committed + .unwrap_or_else(|e| panic!("a read no write moved commits after the leader change: {e}")); cluster.shutdown().await; } diff --git a/nodedb-cluster/src/bootstrap/join.rs b/nodedb-cluster/src/bootstrap/join.rs index 7247608c4..4a7424f3e 100644 --- a/nodedb-cluster/src/bootstrap/join.rs +++ b/nodedb-cluster/src/bootstrap/join.rs @@ -581,7 +581,7 @@ mod tests { ) -> crate::rpc_codec::ShuffleProduceResponse { crate::rpc_codec::ShuffleProduceResponse { error: None, - read_version_lsn: 0, + read_versions: Vec::new(), } } diff --git a/nodedb-cluster/src/calvin/types/transaction.rs b/nodedb-cluster/src/calvin/types/transaction.rs index 0eba9948e..9d68d2b9f 100644 --- a/nodedb-cluster/src/calvin/types/transaction.rs +++ b/nodedb-cluster/src/calvin/types/transaction.rs @@ -494,7 +494,7 @@ mod tests { }; use super::*; use nodedb_types::id::CollectionKey; - use nodedb_types::{KeyRepr, Lsn}; + use nodedb_types::{KeyRepr, WriteVersion}; fn doc_set(collection: &str, surrogates: Vec) -> EngineKeySet { EngineKeySet::Document { @@ -764,17 +764,15 @@ mod tests { engine: EngineTag::Kv, collection: "kv_col".to_owned(), key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(&b"k1"[..]))), - read_lsn: Lsn::new(7), + read_version: WriteVersion::logged(0, 7), home_vshard: None, - served_by: 0, }, VersionedReadEntry { engine: EngineTag::Document, collection: "doc_col".to_owned(), key: ReadKeyIdent::Predicate, - read_lsn: Lsn::new(11), + read_version: WriteVersion::logged(2, 11), home_vshard: None, - served_by: 0, }, ]) } @@ -805,7 +803,7 @@ mod tests { let mut decoded: TxClass = zerompk::from_msgpack(&bytes).expect("decode TxClass"); decoded.restore_derived().expect("restore derived"); - // Every read_lsn and the Point/Predicate distinction survive exactly. + // Every read version and the Point/Predicate distinction survive exactly. assert_eq!(decoded.versioned_reads, reads); assert_eq!(decoded.versioned_reads.len(), 2); let point = decoded @@ -813,7 +811,7 @@ mod tests { .iter() .find(|e| matches!(e.key, ReadKeyIdent::Point(_))) .expect("point entry"); - assert_eq!(point.read_lsn, Lsn::new(7)); + assert_eq!(point.read_version, WriteVersion::logged(0, 7)); assert_eq!( point.key, ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(&b"k1"[..]))) @@ -823,7 +821,7 @@ mod tests { .iter() .find(|e| matches!(e.key, ReadKeyIdent::Predicate)) .expect("predicate entry"); - assert_eq!(predicate.read_lsn, Lsn::new(11)); + assert_eq!(predicate.read_version, WriteVersion::logged(2, 11)); } /// Mirror of `TxClass`'s wire shape from BEFORE `versioned_reads` existed: diff --git a/nodedb-cluster/src/distributed_array/coordinator/read.rs b/nodedb-cluster/src/distributed_array/coordinator/read.rs index fa9ba1597..760c0b184 100644 --- a/nodedb-cluster/src/distributed_array/coordinator/read.rs +++ b/nodedb-cluster/src/distributed_array/coordinator/read.rs @@ -12,8 +12,8 @@ use crate::circuit_breaker::CircuitBreaker; use crate::error::{ClusterError, Result}; use super::super::merge::{ - ArrayAggPartial, any_truncated_before_horizon_agg, merge_slice_rows, merge_slice_rows_sorted, - reduce_agg_partials, + ArrayAggPartial, any_truncated_before_horizon_agg, merge_read_versions, merge_slice_rows, + merge_slice_rows_sorted, reduce_agg_partials, }; use super::super::rpc::ShardRpcDispatch; use super::super::scatter::{FanOutParams, FanOutPartitionedParams, fan_out, fan_out_partitioned}; @@ -50,6 +50,8 @@ pub struct ArrayCoordParams { pub struct CoordSliceResult { pub rows: Vec>, pub truncated_before_horizon: bool, + /// The write versions every shard leg observed, one per vShard. + pub read_versions: Vec, } /// Result of a coordinated aggregate fan-out. @@ -63,6 +65,8 @@ pub struct CoordSliceResult { pub struct CoordAggResult { pub partials: Vec, pub truncated_before_horizon: bool, + /// The write versions every shard leg observed, one per vShard. + pub read_versions: Vec, } /// Compute the inclusive Hilbert-prefix range `[lo, hi]` that vShard `shard_id` @@ -214,6 +218,7 @@ impl ArrayCoordinator { Ok(CoordSliceResult { rows, truncated_before_horizon, + read_versions: merge_read_versions(resps.iter().map(|r| r.read_versions.as_slice())), }) } @@ -263,6 +268,7 @@ impl ArrayCoordinator { Ok(CoordAggResult { partials: reduce_agg_partials(&resps), truncated_before_horizon: any_truncated_before_horizon_agg(&resps), + read_versions: merge_read_versions(resps.iter().map(|r| r.read_versions.as_slice())), }) } @@ -339,6 +345,14 @@ mod tests { use super::*; + /// The version a mock shard leg reports for its vShard. + fn leg_versions(vshard: u32) -> Vec { + vec![nodedb_types::ShardVersion { + vshard, + version: nodedb_types::WriteVersion::logged(1, u64::from(vshard) + 100), + }] + } + /// Mock dispatch that returns a pre-serialised `ArrayShardSliceResp`. struct SliceEchoDispatch { /// Rows to return from each shard. @@ -353,6 +367,7 @@ mod tests { rows_msgpack: self.rows.clone(), truncated: false, truncated_before_horizon: false, + read_versions: leg_versions(req.vshard_id), }; let payload = zerompk::to_msgpack_vec(&resp).unwrap(); Ok(VShardEnvelope::new( @@ -377,6 +392,7 @@ mod tests { shard_id: req.vshard_id, partials: self.partials.clone(), truncated_before_horizon: false, + read_versions: leg_versions(req.vshard_id), }; let payload = zerompk::to_msgpack_vec(&resp).unwrap(); Ok(VShardEnvelope::new( @@ -438,6 +454,16 @@ mod tests { .expect("coord_slice should succeed"); assert_eq!(result.rows.len(), 6); assert!(!result.truncated_before_horizon); + let mut versions = result.read_versions; + versions.sort(); + assert_eq!( + versions, + [0, 1, 2] + .into_iter() + .flat_map(leg_versions) + .collect::>(), + "the merge keeps every shard leg's version" + ); } #[tokio::test] @@ -533,6 +559,7 @@ mod tests { shard_id: req.vshard_id, partials, truncated_before_horizon: false, + read_versions: Vec::new(), }; let payload = zerompk::to_msgpack_vec(&resp).unwrap(); Ok(VShardEnvelope::new( @@ -596,6 +623,7 @@ mod tests { shard_id: req.vshard_id, partials, truncated_before_horizon: below, + read_versions: Vec::new(), }; let payload = zerompk::to_msgpack_vec(&resp).unwrap(); Ok(VShardEnvelope::new( diff --git a/nodedb-cluster/src/distributed_array/handler.rs b/nodedb-cluster/src/distributed_array/handler.rs index 2f8ae0acc..01cafb3e5 100644 --- a/nodedb-cluster/src/distributed_array/handler.rs +++ b/nodedb-cluster/src/distributed_array/handler.rs @@ -78,6 +78,7 @@ async fn handle_slice( rows_msgpack: exec.rows, truncated, truncated_before_horizon: exec.truncated_before_horizon, + read_versions: exec.read_versions, }; serialise(resp) } @@ -98,6 +99,7 @@ async fn handle_agg( shard_id: local_vshard_id, partials: exec.partials, truncated_before_horizon: exec.truncated_before_horizon, + read_versions: exec.read_versions, }; serialise(resp) } @@ -263,6 +265,14 @@ mod tests { }; use super::handle_array_shard_rpc; + /// The version a stub shard read reports for its vShard. + fn stub_versions(vshard: u32) -> Vec { + vec![nodedb_types::ShardVersion { + vshard, + version: nodedb_types::WriteVersion::logged(1, 7), + }] + } + /// Mock executor that returns a fixed set of row bytes for slice, /// a fixed bitmap for surrogate scan, fixed partials for agg, and /// echoes back the `wal_lsn` for put/delete. @@ -278,12 +288,13 @@ mod tests { impl ArrayLocalExecutor for StubExecutor { async fn exec_slice( &self, - _local_vshard_id: u32, + local_vshard_id: u32, _req: &super::super::wire::ArrayShardSliceReq, ) -> Result { Ok(ArraySliceExec { rows: self.rows.clone(), truncated_before_horizon: self.truncated_before_horizon, + read_versions: stub_versions(local_vshard_id), }) } @@ -298,12 +309,13 @@ mod tests { async fn exec_agg( &self, - _local_vshard_id: u32, + local_vshard_id: u32, _req: &ArrayShardAggReq, ) -> Result { Ok(ArrayAggExec { partials: self.partials.clone(), truncated_before_horizon: self.truncated_before_horizon, + read_versions: stub_versions(local_vshard_id), }) } @@ -422,6 +434,34 @@ mod tests { ); } + /// A shard leg that matched no row still reports its vShard's version, + /// and the version survives the response's wire encoding. + #[tokio::test] + async fn an_empty_shard_read_reports_its_version_over_the_wire() { + let executor: Arc = Arc::new(StubExecutor { + rows: vec![], + bitmap: vec![], + partials: vec![], + truncated_before_horizon: false, + affected: 0, + }); + let slice = + handle_array_shard_rpc(ARRAY_SHARD_SLICE_REQ, 5, &make_slice_req_bytes(), &executor) + .await + .expect("slice handler should succeed"); + let slice: ArrayShardSliceResp = + zerompk::from_msgpack(&slice).expect("response should deserialise"); + assert!(slice.rows_msgpack.is_empty()); + assert_eq!(slice.read_versions, stub_versions(5)); + + let agg = handle_array_shard_rpc(ARRAY_SHARD_AGG_REQ, 5, &make_agg_req_bytes(), &executor) + .await + .expect("agg handler should succeed"); + let agg: ArrayShardAggResp = + zerompk::from_msgpack(&agg).expect("response should deserialise"); + assert_eq!(agg.read_versions, stub_versions(5)); + } + #[tokio::test] async fn handle_agg_propagates_truncated_before_horizon() { let executor: Arc = Arc::new(StubExecutor { diff --git a/nodedb-cluster/src/distributed_array/local_executor.rs b/nodedb-cluster/src/distributed_array/local_executor.rs index 13dcf319b..68fcf31ff 100644 --- a/nodedb-cluster/src/distributed_array/local_executor.rs +++ b/nodedb-cluster/src/distributed_array/local_executor.rs @@ -29,6 +29,9 @@ use crate::error::Result; pub struct ArraySliceExec { pub rows: Vec>, pub truncated_before_horizon: bool, + /// The write versions the shard read observed, one per vShard it + /// covers. A transaction validates the read at these versions. + pub read_versions: Vec, } /// Result of a local shard partial aggregate: the per-group partial states @@ -36,6 +39,8 @@ pub struct ArraySliceExec { pub struct ArrayAggExec { pub partials: Vec, pub truncated_before_horizon: bool, + /// The write versions the shard read observed (see [`ArraySliceExec`]). + pub read_versions: Vec, } /// Result of a local shard write (put or delete): the LSN the coordinator diff --git a/nodedb-cluster/src/distributed_array/merge.rs b/nodedb-cluster/src/distributed_array/merge.rs index be00c0cfa..9d11396e7 100644 --- a/nodedb-cluster/src/distributed_array/merge.rs +++ b/nodedb-cluster/src/distributed_array/merge.rs @@ -101,6 +101,22 @@ pub fn any_truncated_before_horizon_agg(shard_resps: &[ArrayShardAggResp]) -> bo shard_resps.iter().any(|r| r.truncated_before_horizon) } +/// The write versions every shard leg observed, one per vShard. Each leg +/// reads its own vShard, so every leg's version is kept. Two legs that report +/// one vShard keep the higher version. +pub fn merge_read_versions<'a>( + legs: impl IntoIterator, +) -> Vec { + let mut merged: Vec = Vec::new(); + for shard in legs.into_iter().flatten() { + match merged.iter_mut().find(|seen| seen.vshard == shard.vshard) { + Some(seen) => seen.version = seen.version.max(shard.version), + None => merged.push(*shard), + } + } + merged +} + /// Merge row batches from multiple shards into one result set. /// /// Rows are concatenated in shard-arrival order (order-independent for @@ -232,11 +248,13 @@ mod tests { shard_id: 0, partials: vec![ArrayAggPartial::from_single(0, 10.0)], truncated_before_horizon: false, + read_versions: Vec::new(), }; let resp_b = ArrayShardAggResp { shard_id: 1, partials: vec![ArrayAggPartial::from_single(0, 20.0)], truncated_before_horizon: false, + read_versions: Vec::new(), }; let merged = reduce_agg_partials(&[resp_a, resp_b]); assert_eq!(merged.len(), 1); @@ -253,6 +271,7 @@ mod tests { ArrayAggPartial::from_single(1, 15.0), ], truncated_before_horizon: false, + read_versions: Vec::new(), }; let merged = reduce_agg_partials(&[resp]); assert_eq!(merged.len(), 2); @@ -277,12 +296,14 @@ mod tests { rows_msgpack: vec![vec![1u8], vec![2u8]], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let r1 = ArrayShardSliceResp { shard_id: 1, rows_msgpack: vec![vec![3u8]], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let rows = merge_slice_rows(&[r0, r1], 0); assert_eq!(rows.len(), 3); @@ -295,6 +316,7 @@ mod tests { rows_msgpack: vec![vec![1u8], vec![2u8], vec![3u8], vec![4u8], vec![5u8]], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let rows = merge_slice_rows(&[resp], 3); assert_eq!(rows.len(), 3); @@ -308,11 +330,13 @@ mod tests { shard_id: 0, partials: vec![ArrayAggPartial::from_single(0, 5.0)], truncated_before_horizon: false, + read_versions: Vec::new(), }; let resp_b = ArrayShardAggResp { shard_id: 1, partials: vec![ArrayAggPartial::from_single(0, 3.0)], truncated_before_horizon: false, + read_versions: Vec::new(), }; let merged = reduce_agg_partials(&[resp_a, resp_b]); assert_eq!(merged.len(), 1); @@ -325,11 +349,13 @@ mod tests { shard_id: 0, partials: vec![ArrayAggPartial::from_single(0, 5.0)], truncated_before_horizon: false, + read_versions: Vec::new(), }; let resp_b = ArrayShardAggResp { shard_id: 1, partials: vec![ArrayAggPartial::from_single(0, 99.0)], truncated_before_horizon: false, + read_versions: Vec::new(), }; let merged = reduce_agg_partials(&[resp_a, resp_b]); assert_eq!(merged.len(), 1); @@ -359,6 +385,7 @@ mod tests { ArrayAggPartial::from_single(1, 10.0), ], truncated_before_horizon: false, + read_versions: Vec::new(), }; let resp_b = ArrayShardAggResp { shard_id: 1, @@ -367,6 +394,7 @@ mod tests { ArrayAggPartial::from_single(2, 30.0), ], truncated_before_horizon: false, + read_versions: Vec::new(), }; let merged = reduce_agg_partials(&[resp_a, resp_b]); assert_eq!(merged.len(), 3); @@ -385,12 +413,14 @@ mod tests { rows_msgpack: vec![], truncated: false, truncated_before_horizon: true, + read_versions: Vec::new(), }; let r1 = ArrayShardSliceResp { shard_id: 1, rows_msgpack: vec![vec![1u8]], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; assert!(any_truncated_before_horizon_slice(&[r0, r1])); @@ -398,11 +428,13 @@ mod tests { shard_id: 0, partials: vec![], truncated_before_horizon: false, + read_versions: Vec::new(), }; let a1 = ArrayShardAggResp { shard_id: 1, partials: vec![], truncated_before_horizon: true, + read_versions: Vec::new(), }; assert!(any_truncated_before_horizon_agg(&[a0, a1])); @@ -410,6 +442,7 @@ mod tests { shard_id: 2, partials: vec![], truncated_before_horizon: false, + read_versions: Vec::new(), }; assert!(!any_truncated_before_horizon_agg(&[a_none])); } @@ -421,11 +454,13 @@ mod tests { shard_id: 0, partials: vec![ArrayAggPartial::from_single(0, 7.0)], truncated_before_horizon: false, + read_versions: Vec::new(), }; let resp_b = ArrayShardAggResp { shard_id: 1, partials: vec![ArrayAggPartial::from_single(1, 13.0)], truncated_before_horizon: false, + read_versions: Vec::new(), }; let merged = reduce_agg_partials(&[resp_a, resp_b]); assert_eq!(merged.len(), 2); @@ -442,12 +477,14 @@ mod tests { rows_msgpack: vec![vec![1u8], vec![2u8]], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let r1 = ArrayShardSliceResp { shard_id: 1, rows_msgpack: vec![vec![3u8], vec![4u8]], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; // Total 4 rows, limit 3 → first 3. let rows = merge_slice_rows(&[r0, r1], 3); @@ -476,12 +513,14 @@ mod tests { rows_msgpack: vec![audit_row(30, 3), audit_row(10, 1)], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let r1 = ArrayShardSliceResp { shard_id: 1, rows_msgpack: vec![audit_row(20, 2)], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let rows = merge_slice_rows_sorted(&[r0, r1], 0).expect("sorted merge"); let times: Vec = rows @@ -499,12 +538,14 @@ mod tests { rows_msgpack: vec![audit_row(50, 5), audit_row(10, 1)], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let r1 = ArrayShardSliceResp { shard_id: 1, rows_msgpack: vec![audit_row(30, 3), audit_row(20, 2)], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let rows = merge_slice_rows_sorted(&[r0, r1], 2).expect("sorted merge"); let times: Vec = rows @@ -523,6 +564,7 @@ mod tests { rows_msgpack: vec![audit_row(10, 1), vec![0xFF, 0xFF, 0xFF]], truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; assert!(merge_slice_rows_sorted(&[resp], 0).is_err()); } @@ -564,8 +606,30 @@ mod tests { rows_msgpack: (0u8..20).map(|i| vec![i]).collect(), truncated: false, truncated_before_horizon: false, + read_versions: Vec::new(), }; let rows = merge_slice_rows(&[resp], 0); assert_eq!(rows.len(), 20); } + + fn shard_version(vshard: u32, index: u64) -> nodedb_types::ShardVersion { + nodedb_types::ShardVersion { + vshard, + version: nodedb_types::WriteVersion::logged(1, index), + } + } + + /// Every leg's version survives the merge, a leg that matched no row + /// included. A vShard two legs report keeps the higher version. + #[test] + fn the_merge_keeps_every_legs_version() { + let legs = [ + vec![shard_version(3, 10)], + Vec::new(), + vec![shard_version(7, 4)], + vec![shard_version(3, 12)], + ]; + let merged = merge_read_versions(legs.iter().map(Vec::as_slice)); + assert_eq!(merged, vec![shard_version(3, 12), shard_version(7, 4)]); + } } diff --git a/nodedb-cluster/src/distributed_array/wire.rs b/nodedb-cluster/src/distributed_array/wire.rs index 0a3611830..2183236bb 100644 --- a/nodedb-cluster/src/distributed_array/wire.rs +++ b/nodedb-cluster/src/distributed_array/wire.rs @@ -72,6 +72,10 @@ pub struct ArrayShardSliceResp { /// True when `system_as_of` is below the oldest tile version on this shard /// and the shard produced zero rows as a result of that horizon. pub truncated_before_horizon: bool, + /// The write versions the shard read observed, one per vShard it covers, + /// even when it matched no row. A transaction validates the read at + /// these versions. + pub read_versions: Vec, } /// Scatter request: coordinator asks a shard to compute a partial aggregate. @@ -118,6 +122,9 @@ pub struct ArrayShardAggResp { /// True when `system_as_of` is below the oldest tile version on this shard /// and the shard produced zero rows as a result of that horizon. pub truncated_before_horizon: bool, + /// The write versions the shard read observed (see + /// [`ArrayShardSliceResp::read_versions`]). + pub read_versions: Vec, } /// Scatter request: coordinator forwards a cell write to the owning shard. diff --git a/nodedb-cluster/src/forward.rs b/nodedb-cluster/src/forward.rs index d6feb1cd2..272b63ffe 100644 --- a/nodedb-cluster/src/forward.rs +++ b/nodedb-cluster/src/forward.rs @@ -20,18 +20,17 @@ use crate::rpc_codec::{ExecuteRequest, ExecuteResponse, TypedClusterError}; /// stream was reset or finished early): the producer MUST stop pulling and /// return without writing a terminal frame. /// -/// `read_version_lsn` is that frame's per-collection read version (its scanned -/// collection's `coll_write_lsn` at read time), distinct from the core-global -/// `watermark_lsn`. The shuffle fan-out sink max-folds it so the producer can -/// report the observed read version for cross-shard OCC validation; the plain -/// `ExecuteStream` sink (which only carries `watermark_lsn` on the wire) ignores -/// it. +/// `read_versions` are that frame's read versions, one per vShard it read, +/// distinct from the core-global `watermark_lsn`. The shuffle fan-out sink +/// folds them so the producer can report what its scan observed for +/// cross-shard OCC validation. The plain `ExecuteStream` sink carries only +/// `watermark_lsn` on the wire and ignores them. pub trait ChunkSink: Send { fn send_chunk( &mut self, payload: Vec, watermark_lsn: u64, - read_version_lsn: u64, + read_versions: Vec, ) -> impl std::future::Future> + Send; } diff --git a/nodedb-cluster/src/raft_loop/handle_rpc/shuffle_calvin.rs b/nodedb-cluster/src/raft_loop/handle_rpc/shuffle_calvin.rs index fee16a61b..ab92b60e3 100644 --- a/nodedb-cluster/src/raft_loop/handle_rpc/shuffle_calvin.rs +++ b/nodedb-cluster/src/raft_loop/handle_rpc/shuffle_calvin.rs @@ -71,7 +71,7 @@ impl RaftLoop { message: "shuffle producer not configured (no ShuffleProducer installed)" .into(), }), - read_version_lsn: 0, + read_versions: Vec::new(), }, } } diff --git a/nodedb-cluster/src/raft_loop/hooks.rs b/nodedb-cluster/src/raft_loop/hooks.rs index ce03e3ca4..0b3c181b3 100644 --- a/nodedb-cluster/src/raft_loop/hooks.rs +++ b/nodedb-cluster/src/raft_loop/hooks.rs @@ -240,9 +240,9 @@ pub trait ShuffleProducer: Send + Sync + 'static { /// Run the local scan fragment, hash-partition its rows, and fan them out to /// the part-owners. Returns a [`ShuffleProduceResponse`] whose `error` is /// `None` on a clean produce or `Some(err)` on a terminal scan failure (after - /// every part has been `End`ed with the error), and whose `read_version_lsn` - /// carries the max per-collection read version the local scan observed (`0` on - /// failure) for the coordinator's cross-shard OCC read validation. + /// every part has been `End`ed with the error), and whose `read_versions` + /// carry the read versions the local scan observed (empty on failure) for + /// the coordinator's cross-shard OCC read validation. async fn on_shuffle_produce( &self, req: crate::rpc_codec::ShuffleProduceRequest, diff --git a/nodedb-cluster/src/rpc_codec/execute/codec.rs b/nodedb-cluster/src/rpc_codec/execute/codec.rs index c156002f1..4ea76e09d 100644 --- a/nodedb-cluster/src/rpc_codec/execute/codec.rs +++ b/nodedb-cluster/src/rpc_codec/execute/codec.rs @@ -222,10 +222,14 @@ mod tests { #[test] fn roundtrip_execute_response_success() { + let read_versions = vec![nodedb_types::ShardVersion { + vshard: 513, + version: nodedb_types::WriteVersion::logged(2, 0xBEEF_5678), + }]; let resp = ExecuteResponse::ok( vec![b"row1".to_vec(), b"row2".to_vec()], 0xCAFE_1234, - 0xBEEF_5678, + read_versions.clone(), ); let decoded = roundtrip_resp(resp); assert!(decoded.success); @@ -237,11 +241,32 @@ mod tests { "read watermark roundtrips on the response body" ); assert_eq!( - decoded.read_version_lsn, 0xBEEF_5678, - "per-collection read-version LSN roundtrips distinct from the watermark" + decoded.read_versions, read_versions, + "read versions roundtrip distinct from the watermark" ); } + #[test] + fn a_refusal_roundtrips_the_versions_it_observed() { + let read_versions = vec![nodedb_types::ShardVersion { + vshard: 7, + version: nodedb_types::WriteVersion::logged(1, 40), + }]; + let resp = ExecuteResponse::refused_with_versions( + TypedClusterError::Internal { + code: 0, + message: "no row".into(), + }, + 12, + read_versions.clone(), + ); + let decoded = roundtrip_resp(resp); + assert!(!decoded.success); + assert!(decoded.error.is_some()); + assert_eq!(decoded.watermark_lsn, 12); + assert_eq!(decoded.read_versions, read_versions); + } + #[test] fn roundtrip_execute_response_not_leader() { let resp = ExecuteResponse::err(TypedClusterError::NotLeader { @@ -256,9 +281,9 @@ mod tests { decoded.watermark_lsn, 0, "error responses carry no watermark" ); - assert_eq!( - decoded.read_version_lsn, 0, - "error responses carry no read-version LSN" + assert!( + decoded.read_versions.is_empty(), + "error responses carry no read versions" ); match decoded.error { Some(TypedClusterError::NotLeader { diff --git a/nodedb-cluster/src/rpc_codec/execute/types.rs b/nodedb-cluster/src/rpc_codec/execute/types.rs index ce85b3aaa..c52225b3f 100644 --- a/nodedb-cluster/src/rpc_codec/execute/types.rs +++ b/nodedb-cluster/src/rpc_codec/execute/types.rs @@ -63,12 +63,12 @@ pub struct ExecuteResponse { /// writes/errors. Mirrors [`ExecuteStreamChunk::watermark_lsn`]: raw `u64` /// on the wire, converted to `Lsn` at the coordinator via `Lsn::new`. pub watermark_lsn: u64, - /// Per-collection read-version LSN for the scanned collection (its - /// `coll_write_lsn` at read time, a WAL LSN); 0 for - /// writes/errors. The sound comparand for cross-shard OCC read validation, - /// distinct from the core-global `watermark_lsn`. Raw `u64` on the wire, - /// converted to `Lsn` at the coordinator via `Lsn::new`. - pub read_version_lsn: u64, + /// The read versions the executing node's cores observed, one per vShard + /// read. A write reports the versions it recorded. Empty for errors other + /// than a read that found no row. The + /// comparand for cross-shard OCC read validation, distinct from the + /// core-global `watermark_lsn`. + pub read_versions: Vec, } /// Typed error returned by the remote executor. @@ -142,22 +142,42 @@ pub struct ExecuteStreamEnd { } impl ExecuteResponse { - pub fn ok(payloads: Vec>, watermark_lsn: u64, read_version_lsn: u64) -> Self { + pub fn ok( + payloads: Vec>, + watermark_lsn: u64, + read_versions: Vec, + ) -> Self { Self { success: true, payloads, error: None, watermark_lsn, - read_version_lsn, + read_versions, } } + /// A refusal that still observed the versions a read validates: a read + /// that found no row. + pub fn refused_with_versions( + error: TypedClusterError, + watermark_lsn: u64, + read_versions: Vec, + ) -> Self { + Self { + success: false, + payloads: vec![], + error: Some(error), + watermark_lsn, + read_versions, + } + } + pub fn err(error: TypedClusterError) -> Self { Self { success: false, payloads: vec![], error: Some(error), watermark_lsn: 0, - read_version_lsn: 0, + read_versions: Vec::new(), } } } diff --git a/nodedb-cluster/src/rpc_codec/shuffle.rs b/nodedb-cluster/src/rpc_codec/shuffle.rs index 379046d5d..76dfa23b6 100644 --- a/nodedb-cluster/src/rpc_codec/shuffle.rs +++ b/nodedb-cluster/src/rpc_codec/shuffle.rs @@ -125,14 +125,12 @@ pub struct ShuffleProduceRequest { #[derive(Debug, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] pub struct ShuffleProduceResponse { pub error: Option, - /// Max per-collection read-version LSN observed by the producer's local scan - /// (its scanned collection's `coll_write_lsn` at read time, a WAL LSN); 0 for - /// a failed produce. The sound comparand the coordinator max-folds across + /// The read versions the producer's local scan observed, one per vShard + /// read. Empty for a failed produce. The coordinator folds them across /// producers for cross-shard OCC read validation of an in-transaction - /// distributed aggregate — distinct from the core-global watermark. Raw `u64` - /// on the wire, converted to `Lsn` at the coordinator via `Lsn::new`. Mirrors - /// [`ExecuteResponse::read_version_lsn`](super::execute::ExecuteResponse::read_version_lsn). - pub read_version_lsn: u64, + /// distributed aggregate. Mirrors + /// [`ExecuteResponse::read_versions`](super::execute::ExecuteResponse::read_versions). + pub read_versions: Vec, } /// One `(left_key, right_key)` equi-join pair of a [`ShuffleConsumeRequest`]'s @@ -639,14 +637,18 @@ mod tests { #[test] fn roundtrip_shuffle_produce_response_clean() { + let read_versions = vec![nodedb_types::ShardVersion { + vshard: 9, + version: nodedb_types::WriteVersion::logged(1, 0xABCD_1234), + }]; let decoded = roundtrip_produce_resp(ShuffleProduceResponse { error: None, - read_version_lsn: 0xABCD_1234, + read_versions: read_versions.clone(), }); assert!(decoded.error.is_none()); assert_eq!( - decoded.read_version_lsn, 0xABCD_1234, - "producer read-version LSN roundtrips on the produce reply" + decoded.read_versions, read_versions, + "producer read versions roundtrip on the produce reply" ); } @@ -657,11 +659,11 @@ mod tests { code: 0x55, message: "produce scan failed".into(), }), - read_version_lsn: 0, + read_versions: Vec::new(), }); - assert_eq!( - decoded.read_version_lsn, 0, - "a failed produce carries no read-version LSN" + assert!( + decoded.read_versions.is_empty(), + "a failed produce carries no read versions" ); match decoded.error { Some(TypedClusterError::Internal { code, message }) => { diff --git a/nodedb-cluster/src/transport/client/transport.rs b/nodedb-cluster/src/transport/client/transport.rs index 84cdbe740..74c569bf7 100644 --- a/nodedb-cluster/src/transport/client/transport.rs +++ b/nodedb-cluster/src/transport/client/transport.rs @@ -512,7 +512,7 @@ pub(super) mod tests { // Emit three deterministic chunks then clean EOF. Exercises the // multi-frame chunk/end envelope path end-to-end over loopback QUIC. for i in 0u64..3 { - if let Err(_e) = sink.send_chunk(vec![i as u8; 4], i + 100, 0).await { + if let Err(_e) = sink.send_chunk(vec![i as u8; 4], i + 100, Vec::new()).await { // Coordinator gone — stop producing, no terminal frame. return None; } @@ -547,7 +547,7 @@ pub(super) mod tests { ) -> crate::rpc_codec::ShuffleProduceResponse { crate::rpc_codec::ShuffleProduceResponse { error: None, - read_version_lsn: 0, + read_versions: Vec::new(), } } diff --git a/nodedb-cluster/src/transport/rpc_handler.rs b/nodedb-cluster/src/transport/rpc_handler.rs index 385fee368..13d2e4c9d 100644 --- a/nodedb-cluster/src/transport/rpc_handler.rs +++ b/nodedb-cluster/src/transport/rpc_handler.rs @@ -71,8 +71,8 @@ pub trait RaftRpcHandler: Send + Sync + 'static { /// (looping back for parts this node owns). Returns a /// [`ShuffleProduceResponse`] whose `error` is `None` on a clean produce or /// `Some(err)` if the scan failed (every part has already been `End`ed with - /// the same error), and whose `read_version_lsn` carries the max - /// per-collection read version the scan observed. The transport writes + /// the same error), and whose `read_versions` carry the read versions the + /// scan observed, one per vShard. The transport writes /// exactly one `ShuffleProduceResponse` carrying this outcome back to the /// coordinator. fn on_shuffle_produce( diff --git a/nodedb-cluster/src/transport/server.rs b/nodedb-cluster/src/transport/server.rs index f694fb2d1..a52417851 100644 --- a/nodedb-cluster/src/transport/server.rs +++ b/nodedb-cluster/src/transport/server.rs @@ -72,10 +72,10 @@ impl ChunkSink for QuicChunkSink<'_> { &mut self, payload: Vec, watermark_lsn: u64, - // The `ExecuteStream` wire chunk only carries `watermark_lsn`; the - // per-collection read version is surfaced separately on the shuffle - // produce reply, not on this streaming path. - _read_version_lsn: u64, + // The `ExecuteStream` wire chunk only carries `watermark_lsn`. The + // read versions reach the coordinator on the shuffle produce reply, + // not on this streaming path. + _read_versions: Vec, ) -> Result<()> { let rpc = RaftRpc::ExecuteStreamChunk(ExecuteStreamChunk { payload, diff --git a/nodedb-physical/src/physical_plan/meta.rs b/nodedb-physical/src/physical_plan/meta.rs index 8f29bffbc..8fafaa00f 100644 --- a/nodedb-physical/src/physical_plan/meta.rs +++ b/nodedb-physical/src/physical_plan/meta.rs @@ -132,6 +132,12 @@ pub enum MetaOp { /// their edges too. #[serde(default)] group_vshards: Vec, + /// The version each of the group's vShards holds once the install + /// lands: the log position of the snapshot's cut. The installed rows + /// carry no per-row versions, so a read older than the cut no longer + /// validates on this node. Empty = no group install. + #[serde(default)] + version_floor: Vec, }, /// Purge ALL data for a tenant across every engine and cache. diff --git a/nodedb-physical/src/physical_plan/meta_home.rs b/nodedb-physical/src/physical_plan/meta_home.rs index cec6fb3d0..ff9c52950 100644 --- a/nodedb-physical/src/physical_plan/meta_home.rs +++ b/nodedb-physical/src/physical_plan/meta_home.rs @@ -53,9 +53,9 @@ pub struct HomeVersion { zerompk::FromMessagePack, )] pub enum HomeAnswer { - /// The collection's write floor on the probe's core, or that core's - /// watermark for a probe with no collection. - Version(u64), + /// The collection's write floor on the probe's vShard, or that vShard's + /// latest version for a probe with no collection. + Version(nodedb_types::WriteVersion), /// The node does not hold the leader lease of the probe's group, so it /// cannot answer. `leader_node` is the leader its routing table names at /// `leader_term`, `0` when it names none. @@ -74,7 +74,7 @@ mod tests { vshard: 7, collection: Some("db1.edges".into()), }, - answer: HomeAnswer::Version(42), + answer: HomeAnswer::Version(nodedb_types::WriteVersion::logged(3, 42)), }, HomeVersion { probe: HomeVersionProbe { diff --git a/nodedb-test-support/src/cluster_harness/node/inspect/core_placement.rs b/nodedb-test-support/src/cluster_harness/node/inspect/core_placement.rs index 888661475..24cd79998 100644 --- a/nodedb-test-support/src/cluster_harness/node/inspect/core_placement.rs +++ b/nodedb-test-support/src/cluster_harness/node/inspect/core_placement.rs @@ -84,6 +84,7 @@ impl TestClusterNode { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Exempt( nodedb::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb-test-support/src/cluster_harness/node/inspect/crdt.rs b/nodedb-test-support/src/cluster_harness/node/inspect/crdt.rs index 24df83b47..d55e8aba2 100644 --- a/nodedb-test-support/src/cluster_harness/node/inspect/crdt.rs +++ b/nodedb-test-support/src/cluster_harness/node/inspect/crdt.rs @@ -57,6 +57,7 @@ impl TestClusterNode { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Exempt( nodedb::bridge::envelope::ExemptReason::Read, ), @@ -131,6 +132,7 @@ impl TestClusterNode { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, }; diff --git a/nodedb-test-support/src/cluster_harness/node/inspect/snapshot.rs b/nodedb-test-support/src/cluster_harness/node/inspect/snapshot.rs index 1a25df725..17d79e170 100644 --- a/nodedb-test-support/src/cluster_harness/node/inspect/snapshot.rs +++ b/nodedb-test-support/src/cluster_harness/node/inspect/snapshot.rs @@ -80,6 +80,7 @@ impl TestClusterNode { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Exempt( nodedb::bridge::envelope::ExemptReason::Read, ), @@ -133,6 +134,7 @@ impl TestClusterNode { replace_mode: true, collections_to_clear: Vec::new(), group_vshards: Vec::new(), + version_floor: Vec::new(), }), deadline: std::time::Instant::now() + std::time::Duration::from_secs(5), priority: Priority::Normal, @@ -147,6 +149,7 @@ impl TestClusterNode { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Exempt( nodedb::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb-test-support/src/cluster_harness/node/inspect/timeseries.rs b/nodedb-test-support/src/cluster_harness/node/inspect/timeseries.rs index e16519549..0b922c5ca 100644 --- a/nodedb-test-support/src/cluster_harness/node/inspect/timeseries.rs +++ b/nodedb-test-support/src/cluster_harness/node/inspect/timeseries.rs @@ -68,6 +68,7 @@ impl TestClusterNode { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Exempt( nodedb::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb-test-support/src/tx_batch_helpers.rs b/nodedb-test-support/src/tx_batch_helpers.rs index ca8b2fa46..d99205fda 100644 --- a/nodedb-test-support/src/tx_batch_helpers.rs +++ b/nodedb-test-support/src/tx_batch_helpers.rs @@ -62,6 +62,7 @@ pub fn make_request(plan: PhysicalPlan) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb-types/src/calvin.rs b/nodedb-types/src/calvin.rs index c7b9c9208..5b214bfe0 100644 --- a/nodedb-types/src/calvin.rs +++ b/nodedb-types/src/calvin.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; -use crate::{KeyRepr, Lsn}; +use crate::{KeyRepr, WriteVersion}; /// A newtype over `Vec` that guarantees sorted, deduplicated contents. /// @@ -299,13 +299,13 @@ pub enum ReadKeyIdent { }, } -/// One LSN-versioned, predicate-aware read observed by a transaction, carried -/// on the replicated Calvin `TxClass` so participants can validate it at the +/// One versioned, predicate-aware read observed by a transaction, carried on +/// the replicated Calvin `TxClass` so participants can validate it at the /// commit serialization point. /// -/// `read_lsn` is the responding shard's write-LSN watermark at read time. The -/// enclosing `TxClass` scopes the tenant; per-database scoping is carried by -/// the transaction as a whole. +/// `read_version` is a data-group log position, so any replica of the read's +/// vShard validates it. The enclosing `TxClass` scopes the tenant; +/// per-database scoping is carried by the transaction as a whole. #[derive( Debug, Clone, @@ -323,21 +323,17 @@ pub struct VersionedReadEntry { pub collection: String, /// Point-key or collection-scoped-predicate identity of the observation. pub key: ReadKeyIdent, - /// The responding shard's write-LSN watermark at read time. - pub read_lsn: Lsn, + /// The version of what the read observed on its home vShard. + pub read_version: WriteVersion, /// The vShard whose write versions validate this read. `None` homes the /// read to its collection's vShard. A graph read names the key vShard it /// read edges on, because edges live on their endpoints' vShards. A homed /// read with an empty `collection` observed every collection there, so it - /// validates against the shard's core watermark. + /// validates against the vShard's latest version. pub home_vshard: Option, - /// The node that served the read. `read_lsn` is a position in that node's - /// WAL, so a participant on any other node treats the read as changed. - /// `0` when no one node is known to have served it. - pub served_by: u64, } -/// The LSN-versioned read-set of a Calvin transaction. +/// The versioned read-set of a Calvin transaction. /// /// Empty for pure-write transactions and for autocommit statements (which /// accumulate no session read-set). Populated at commit time from the neutral diff --git a/nodedb-types/src/lib.rs b/nodedb-types/src/lib.rs index a1a207475..64c83b675 100644 --- a/nodedb-types/src/lib.rs +++ b/nodedb-types/src/lib.rs @@ -73,6 +73,7 @@ pub mod vector_index_stats; pub mod vector_model; pub mod volatility; pub mod wire_version; +pub mod write_version; pub use approx::{CountMinSketch, HyperLogLog, SpaceSaving, TDigest}; pub use array_cell::ArrayCell; @@ -150,3 +151,4 @@ pub use vector_index_stats::{ }; pub use vector_model::{VectorModelEntry, VectorModelMetadata}; pub use volatility::Volatility; +pub use write_version::{ShardVersion, WriteVersion}; diff --git a/nodedb-types/src/write_version.rs b/nodedb-types/src/write_version.rs new file mode 100644 index 000000000..0dc0d05d4 --- /dev/null +++ b/nodedb-types/src/write_version.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The version a committed write gives the rows it writes. +//! +//! A write that applies a data-group Raft entry takes the entry's log +//! position. Every replica applies the entry at the same position, so a +//! version read on one replica compares with the versions every other +//! replica holds. + +use serde::{Deserialize, Serialize}; + +/// The position of the write that produced a row version, within one vShard. +/// +/// A write that applies a data-group entry records `epoch`, the epoch at +/// which its vShard moved to the entry's group, and `index`, the entry's log +/// index. `local` is `0`. +/// +/// A write that applies no entry records its WAL LSN in `local`, on top of +/// the latest version its vShard holds on the core. Such a write exists on +/// one node only: a single-node write, or a local-only write on a cluster +/// node. It sorts after every version its vShard already holds and before +/// the next entry the vShard applies. +/// +/// Versions compare only within one vShard. The writes of one vShard are +/// totally ordered by `(epoch, index, local)`. +#[derive( + Debug, + Clone, + Copy, + Default, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct WriteVersion { + /// The epoch at which the vShard moved to the group that applied the + /// write. `0` while the vShard stays in its initial group. + pub epoch: u64, + /// The log index of the data-group entry that applied the write. + pub index: u64, + /// The WAL LSN of a write that applies no entry, `0` otherwise. + pub local: u64, +} + +impl WriteVersion { + /// The version below every write. + pub const ZERO: Self = Self { + epoch: 0, + index: 0, + local: 0, + }; + + /// The version of a write that applies the entry at `index`, in `epoch`. + pub const fn logged(epoch: u64, index: u64) -> Self { + Self { + epoch, + index, + local: 0, + } + } + + /// The version of a write that applies no entry, at WAL LSN `lsn`, on a + /// vShard whose latest version is `latest`. + pub const fn local_after(latest: Self, lsn: u64) -> Self { + Self { + epoch: latest.epoch, + index: latest.index, + local: lsn, + } + } + + /// How far `self` sits above `older` in one epoch: the log entries + /// between them, or the WAL LSNs between two writes of one entry + /// position. `None` across epochs, or when `older` is not below `self`. + pub fn distance_above(self, older: Self) -> Option { + if self.epoch != older.epoch || self <= older { + return None; + } + if self.index > older.index { + return Some(self.index - older.index); + } + Some(self.local.saturating_sub(older.local)) + } +} + +/// One vShard's write version. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct ShardVersion { + /// The vShard whose history positions `version`. + pub vshard: u32, + pub version: WriteVersion, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_later_entry_sorts_after_a_local_write_of_the_earlier_one() { + let entry = WriteVersion::logged(0, 10); + let local = WriteVersion::local_after(entry, 900); + let next = WriteVersion::logged(0, 11); + assert!(entry < local); + assert!(local < next); + } + + #[test] + fn a_later_epoch_sorts_after_every_index_of_an_earlier_one() { + assert!(WriteVersion::logged(0, 1_000) < WriteVersion::logged(5, 1)); + } + + #[test] + fn distance_counts_entries_then_wal_lsns_within_one_epoch() { + let base = WriteVersion::logged(2, 10); + assert_eq!(WriteVersion::logged(2, 15).distance_above(base), Some(5)); + let local = WriteVersion::local_after(base, 40); + assert_eq!( + WriteVersion::local_after(base, 90).distance_above(local), + Some(50) + ); + assert_eq!(WriteVersion::logged(3, 1).distance_above(base), None); + assert_eq!(base.distance_above(WriteVersion::logged(2, 12)), None); + } + + #[test] + fn versions_round_trip_through_msgpack() { + let version = ShardVersion { + vshard: 7, + version: WriteVersion { + epoch: 3, + index: 41, + local: 9, + }, + }; + let bytes = zerompk::to_msgpack_vec(&version).expect("encode"); + let decoded: ShardVersion = zerompk::from_msgpack(&bytes).expect("decode"); + assert_eq!(decoded, version); + } +} diff --git a/nodedb/src/bridge/admission_chokepoint.rs b/nodedb/src/bridge/admission_chokepoint.rs index e1d775ef3..fde8582da 100644 --- a/nodedb/src/bridge/admission_chokepoint.rs +++ b/nodedb/src/bridge/admission_chokepoint.rs @@ -145,6 +145,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission, } } diff --git a/nodedb/src/bridge/dispatch/drain.rs b/nodedb/src/bridge/dispatch/drain.rs index 22b9fdf0b..de930e493 100644 --- a/nodedb/src/bridge/dispatch/drain.rs +++ b/nodedb/src/bridge/dispatch/drain.rs @@ -169,7 +169,7 @@ impl Dispatcher { ), })), stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }); } diff --git a/nodedb/src/bridge/dispatch/enqueue.rs b/nodedb/src/bridge/dispatch/enqueue.rs index 31ddff03f..5716f6aaf 100644 --- a/nodedb/src/bridge/dispatch/enqueue.rs +++ b/nodedb/src/bridge/dispatch/enqueue.rs @@ -323,7 +323,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) @@ -388,7 +388,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) diff --git a/nodedb/src/bridge/dispatch/response_poll.rs b/nodedb/src/bridge/dispatch/response_poll.rs index 54673aa25..29e2738a9 100644 --- a/nodedb/src/bridge/dispatch/response_poll.rs +++ b/nodedb/src/bridge/dispatch/response_poll.rs @@ -94,7 +94,7 @@ impl Dispatcher { ), })), stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }); } @@ -134,7 +134,7 @@ mod tests { watermark_lsn: Lsn::new(42), error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) @@ -165,7 +165,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) @@ -294,7 +294,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) @@ -389,7 +389,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, } diff --git a/nodedb/src/bridge/dispatch/test_requests.rs b/nodedb/src/bridge/dispatch/test_requests.rs index efb02d80d..f2a8592db 100644 --- a/nodedb/src/bridge/dispatch/test_requests.rs +++ b/nodedb/src/bridge/dispatch/test_requests.rs @@ -39,6 +39,7 @@ pub(super) fn make_request(vshard: u32) -> envelope::Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), } } @@ -72,6 +73,7 @@ pub(super) fn make_request_for_db(vshard: u32, db: u64, req_id: u64) -> envelope wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), } } diff --git a/nodedb/src/bridge/envelope/request.rs b/nodedb/src/bridge/envelope/request.rs index 2c2c29e8d..b1e074483 100644 --- a/nodedb/src/bridge/envelope/request.rs +++ b/nodedb/src/bridge/envelope/request.rs @@ -110,6 +110,13 @@ pub struct Request { /// of the WAL record they reproduce. pub commit_hlc: Option, + /// The log position of the data-group Raft entry this write applies. The + /// Data Plane records it as the version of every row the write touches, + /// so every replica that applies the entry records the same version. + /// `None` for reads, control ops, and writes that apply no entry: those + /// version by their WAL LSN on the core. + pub entry_version: Option, + /// Write-admission decision for this request. /// /// Every write-class [`PhysicalPlan`] MUST pass the neutral write-admission @@ -232,6 +239,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), } } @@ -287,6 +295,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }; match req.plan { diff --git a/nodedb/src/bridge/envelope/response.rs b/nodedb/src/bridge/envelope/response.rs index 378d90f2c..ea1a53c39 100644 --- a/nodedb/src/bridge/envelope/response.rs +++ b/nodedb/src/bridge/envelope/response.rs @@ -193,16 +193,17 @@ pub struct Response { /// Watermark LSN at the time of read (for snapshot consistency tracking). pub watermark_lsn: Lsn, - /// Per-collection read-version LSN (the scanned collection's `coll_write_lsn` - /// at read time, a WAL LSN) — the sound comparand for cross-shard OCC read - /// validation. Distinct from `watermark_lsn` (core-global max, used for - /// snapshot/SI reporting). + /// The write versions the read observed, one per vShard: the read + /// collection's version on its home vShard, or every vShard's latest + /// version a graph read walked. The comparand for cross-shard OCC read + /// validation. Distinct from `watermark_lsn` (this core's WAL position). /// /// On a WRITE response it is the POST-write version of the written - /// collection (the handlers record before responding), which is how the Raft - /// apply path returns a committed write's own version to its proposer. - /// `Lsn::ZERO` when the plan names no single user collection. - pub read_version_lsn: Lsn, + /// collection on the write's vShard (the handlers record before + /// responding), which is how the Raft apply path returns a committed + /// write's own version to its proposer. Empty when the plan names no + /// user collection. + pub read_versions: crate::types::ReadVersions, /// Error code if status is not Ok. pub error_code: Option>, @@ -234,7 +235,7 @@ mod tests { watermark_lsn: Lsn::new(42), error_code: None, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }; assert_eq!(resp.status, Status::Ok); @@ -253,7 +254,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: Some(Box::new(ErrorCode::DeadlineExceeded)), stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }; assert_eq!( diff --git a/nodedb/src/control/array_catalog/ddl.rs b/nodedb/src/control/array_catalog/ddl.rs index d17b60428..328b02c0d 100644 --- a/nodedb/src/control/array_catalog/ddl.rs +++ b/nodedb/src/control/array_catalog/ddl.rs @@ -100,7 +100,7 @@ pub(crate) async fn run_trusted_array_ddl( watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }) } diff --git a/nodedb/src/control/array_sync/raft_apply/common.rs b/nodedb/src/control/array_sync/raft_apply/common.rs index 80db089b4..140ec0c1a 100644 --- a/nodedb/src/control/array_sync/raft_apply/common.rs +++ b/nodedb/src/control/array_sync/raft_apply/common.rs @@ -126,12 +126,18 @@ pub(super) async fn submit_array_write( // committed plan: the proposer's LSN is deliberately not carried on // the wire, and the array engine's tile state has no other // durability path than this record's replay. - // An array write emits no Data-Plane change event to position. + // The entry's position versions the write's rows alike on every + // replica, and its marker gives restart replay the same version. durability: WalDurability::AppendHere { now_override: resolved_now_ms, apply_key, commit_hlc, - change_position: None, + change_position: Some(crate::event::cdc::position::entry_position( + state, + vshard.as_u32(), + group_id, + log_index, + )), }, // Raft committed this entry at a fixed log index and every replica // applies it in that order; re-entering the write-admission gate @@ -296,6 +302,7 @@ pub(super) fn build_array_request( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), @@ -352,7 +359,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/backup/restore/durable.rs b/nodedb/src/control/backup/restore/durable.rs index 7fb39e1cb..3fc14e7d1 100644 --- a/nodedb/src/control/backup/restore/durable.rs +++ b/nodedb/src/control/backup/restore/durable.rs @@ -68,13 +68,13 @@ pub(crate) async fn reissue_plan_durably( .with_event_source(crate::event::EventSource::Restore) .with_restore_id(target.restore_id); let deadline = crate::control::wal_replication::statement_propose_deadline(state); - let (_, write_version) = + let (_, write_versions) = crate::control::wal_replication::propose_replicated_entry(state, proposer, entry, deadline) .await?; tracing::debug!( collection, vshard_id = vshard.as_u32(), - write_version = write_version.as_u64(), + write_version = ?write_versions.of(vshard), "restore: re-issued write applied on this node" ); Ok(()) diff --git a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs index 58d4b9a83..d71b8f6bf 100644 --- a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs +++ b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/array.rs @@ -211,7 +211,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) diff --git a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/core_fanout.rs b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/core_fanout.rs index 24e933ee2..569aba8a0 100644 --- a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/core_fanout.rs +++ b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/core_fanout.rs @@ -146,6 +146,7 @@ pub(super) fn send_to_every_core( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), @@ -225,7 +226,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/crdt_compact.rs b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/crdt_compact.rs index 87322a30d..05c23a2ae 100644 --- a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/crdt_compact.rs +++ b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/crdt_compact.rs @@ -189,7 +189,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) diff --git a/nodedb/src/control/checkpoint_manager.rs b/nodedb/src/control/checkpoint_manager.rs index 86e27c30d..40b0eabde 100644 --- a/nodedb/src/control/checkpoint_manager.rs +++ b/nodedb/src/control/checkpoint_manager.rs @@ -254,6 +254,7 @@ pub async fn run_checkpoint_cycle(inputs: CheckpointCycleInputs<'_>) -> Option { - StagedVote::SerializationConflict - } Ok(vote) => vote, Err(error) => { self.metrics.record_executor_error(); @@ -145,22 +140,6 @@ impl Scheduler { self.resume_on_verdict(txn_id, verdict); } } - - /// Whether a read this vShard validates for `txn_id` was served by a node - /// other than this one. Its `read_lsn` is a position in that node's WAL. - fn validates_read_served_elsewhere(&self, txn_id: TxnId) -> bool { - let Some(pending) = self.pending.get(&txn_id) else { - return false; - }; - let tx_class = &pending.txn.tx_class; - tx_class.versioned_reads.iter().any(|entry| { - super::super::routing::versioned_read_homes_on( - entry, - tx_class.database_id, - self.vshard_id, - ) && entry.served_by != self.shared.node_id - }) - } } #[cfg(test)] diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/request.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/request.rs index 57f186302..11cf941ab 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/request.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/request.rs @@ -76,6 +76,7 @@ impl Scheduler { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::AlreadyOrdered), } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/staged_vote.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/staged_vote.rs index 398ab057d..165864f9f 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/staged_vote.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/staged_vote.rs @@ -112,7 +112,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs index a30c8db79..764cc57fa 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs @@ -199,9 +199,8 @@ pub(super) fn make_validate_only_txn(epoch: u64, position: u32) -> SequencedTxn engine: EngineTag::Document, collection: "test_coll".to_string(), key: ReadKeyIdent::Point(KeyRepr::Surrogate(1)), - read_lsn: Lsn::ZERO, + read_version: nodedb_types::WriteVersion::ZERO, home_vshard: None, - served_by: 0, }]); let tx_class = TxClass::new_single_vshard( ReadWriteSet::new(vec![]), @@ -297,6 +296,7 @@ fn filler_request(request_id: RequestId, tenant_id: TenantId) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), } } @@ -487,7 +487,7 @@ pub(super) fn staged_response(status: Status, stage_vote: Option) -> watermark_lsn: Lsn::ZERO, error_code: None, stage_vote, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/cluster/snapshot_applier.rs b/nodedb/src/control/cluster/snapshot_applier.rs index 035c04813..76d1def3e 100644 --- a/nodedb/src/control/cluster/snapshot_applier.rs +++ b/nodedb/src/control/cluster/snapshot_applier.rs @@ -50,7 +50,7 @@ use crate::bridge::envelope::PhysicalPlan; use crate::control::cluster::snapshot_install::{ CoreMap, CoreShares, SettleStep, SnapshotInstallError, append_install_barrier, append_install_marker, clear_targets_per_core, group_collections, install_on_every_core, - split_by_core, + split_by_core, version_floor, }; use crate::control::state::SharedState; use crate::types::{SurrogateBindEntry, TenantDataSnapshot, TenantId, VShardId}; @@ -180,6 +180,7 @@ impl DataPlaneSnapshotApplier { // vShards. A store with none of them stays as it is. let mut vshards: Vec = group_vshards.iter().copied().collect(); vshards.sort_unstable(); + let version_floor = version_floor(&self.shared, group_id, &vshards, cut_index); let plans = per_core .into_iter() @@ -198,6 +199,7 @@ impl DataPlaneSnapshotApplier { replace_mode: true, collections_to_clear, group_vshards: vshards.clone(), + version_floor: version_floor.clone(), })) }) .collect::, SnapshotInstallError>>()?; diff --git a/nodedb/src/control/cluster/snapshot_install/dispatch.rs b/nodedb/src/control/cluster/snapshot_install/dispatch.rs index c894970ca..762918c57 100644 --- a/nodedb/src/control/cluster/snapshot_install/dispatch.rs +++ b/nodedb/src/control/cluster/snapshot_install/dispatch.rs @@ -58,6 +58,7 @@ pub async fn install_on_every_core( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::AlreadyOrdered), }; let rx = state.tracker.register(request_id); diff --git a/nodedb/src/control/cluster/snapshot_install/mod.rs b/nodedb/src/control/cluster/snapshot_install/mod.rs index 10e49e144..073ef3175 100644 --- a/nodedb/src/control/cluster/snapshot_install/mod.rs +++ b/nodedb/src/control/cluster/snapshot_install/mod.rs @@ -7,9 +7,11 @@ pub mod clear; pub mod dispatch; pub mod error; pub mod split; +pub mod version_floor; pub use barrier::{append_install_barrier, append_install_marker}; pub use clear::{GroupCollection, clear_targets_per_core, group_collections}; pub use dispatch::install_on_every_core; pub use error::{SettleStep, SnapshotInstallError}; pub use split::{CoreMap, CoreShares, split_by_core}; +pub use version_floor::version_floor; diff --git a/nodedb/src/control/cluster/snapshot_install/version_floor.rs b/nodedb/src/control/cluster/snapshot_install/version_floor.rs new file mode 100644 index 000000000..d1b97b231 --- /dev/null +++ b/nodedb/src/control/cluster/snapshot_install/version_floor.rs @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The write version a data-group snapshot install lands on each vShard. + +use nodedb_types::{ShardVersion, WriteVersion}; + +use crate::control::state::SharedState; + +/// The version each of `vshards` holds once a snapshot of `group_id` cut at +/// `cut_index` lands: the cut's log position in the vShard's epoch. +/// +/// The installed rows carry no per-row versions. Every write the snapshot +/// holds sits at or below the cut, so a read the cut does not cover no longer +/// validates on this node. +pub fn version_floor( + state: &SharedState, + group_id: u64, + vshards: &[u32], + cut_index: u64, +) -> Vec { + vshards + .iter() + .map(|&vshard| { + let position = + crate::event::cdc::position::entry_position(state, vshard, group_id, cut_index); + ShardVersion { + vshard, + version: WriteVersion::logged(position.epoch, position.log_index), + } + }) + .collect() +} diff --git a/nodedb/src/control/cluster/start_raft/proposer_wiring.rs b/nodedb/src/control/cluster/start_raft/proposer_wiring.rs index b45846901..f83b79065 100644 --- a/nodedb/src/control/cluster/start_raft/proposer_wiring.rs +++ b/nodedb/src/control/cluster/start_raft/proposer_wiring.rs @@ -291,16 +291,9 @@ fn install_async_proposer( e } }) - // Carry out the write-version the APPLY side stamped, not - // `log_index`. The tracker resolves on the node that applied - // the entry locally, so `write_version` is this replica's own - // post-write `coll_write_lsn` — a WAL LSN, the same domain - // every other feed of that map records in, and the only - // domain the shard-local OCC read validator compares in. The - // raft log index is a per-group counter on a different scale - // entirely; publishing it here made reads validate a WAL LSN - // against a log index. - .map(|applied| (applied.payload, applied.write_version)); + // Carry out the versions the apply side stamped: the + // entry's log position on every vShard the write touched. + .map(|applied| (applied.payload, applied.write_versions)); let applied = applied?; // A write to a vShard homing a permission-tree source is // acknowledged only once every lease holder covers it, or its diff --git a/nodedb/src/control/cluster/start_raft/sequencer_route.rs b/nodedb/src/control/cluster/start_raft/sequencer_route.rs index 1209a2336..e182e2d12 100644 --- a/nodedb/src/control/cluster/start_raft/sequencer_route.rs +++ b/nodedb/src/control/cluster/start_raft/sequencer_route.rs @@ -57,9 +57,11 @@ pub(super) fn routed_write(state: &Weak, data: Vec) -> Proposed /// The apply result a routed write's answer stands for: its payload and /// written version, or its typed Data-Plane verdict. -fn applied_result(response: &Response) -> crate::Result<(Vec, crate::types::Lsn)> { +fn applied_result( + response: &Response, +) -> crate::Result { if response.status == Status::Ok { - return Ok((response.payload.to_vec(), response.read_version_lsn)); + return Ok((response.payload.to_vec(), response.read_versions.clone())); } match response.error_code.as_deref() { Some(code) => Err(crate::Error::DataPlane(code.clone())), @@ -80,10 +82,13 @@ mod tests { fn an_ok_answer_carries_its_payload_and_version() { let mut response = bare_ok_response(RequestId::new(1)); response.payload = crate::bridge::envelope::Payload::from_vec(vec![7]); - response.read_version_lsn = crate::types::Lsn::new(9); - let (payload, version) = applied_result(&response).expect("ok"); + let version = nodedb_types::WriteVersion::logged(2, 9); + response + .read_versions + .note(crate::types::VShardId::new(4), version); + let (payload, versions) = applied_result(&response).expect("ok"); assert_eq!(payload, vec![7]); - assert_eq!(version, crate::types::Lsn::new(9)); + assert_eq!(versions.of(crate::types::VShardId::new(4)), Some(version)); } #[test] diff --git a/nodedb/src/control/crdt_admission.rs b/nodedb/src/control/crdt_admission.rs index 517d1c9df..560ff47ed 100644 --- a/nodedb/src/control/crdt_admission.rs +++ b/nodedb/src/control/crdt_admission.rs @@ -54,7 +54,9 @@ pub struct CrdtApplyAdmissionRequest<'a> { pub struct CrdtAdmissionOutcome { pub payload: Vec, - pub write_version: crate::types::Lsn, + /// The versions the apply stamped: the entry's log position on the + /// written vShard. + pub write_versions: crate::types::ReadVersions, /// Operations the admitted delta encoded that the target document already /// knew, measured by the preview that fenced this apply. Distinguishes a /// client whose writes land from one whose writes are being absorbed — @@ -597,7 +599,7 @@ async fn apply_fenced( // This node's apply of the entry recorded its commit HLC. Ok(CrdtAdmissionOutcome { payload: outcome.0, - write_version: outcome.1, + write_versions: outcome.1, trimmed_ops: 0, }) } @@ -684,7 +686,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -786,7 +788,7 @@ mod tests { .expect("fences lock") .push(expected_frontier_digest); } - Ok((Vec::new(), Lsn::ZERO)) + Ok((Vec::new(), crate::types::ReadVersions::new())) }) }); crate::control::vshard_admission::install_async_raft_proposer( @@ -885,7 +887,7 @@ mod tests { } other => panic!("unexpected replicated write: {other:?}"), } - Ok((Vec::new(), Lsn::ZERO)) + Ok((Vec::new(), crate::types::ReadVersions::new())) }) }); crate::control::vshard_admission::install_async_raft_proposer( @@ -935,7 +937,7 @@ mod tests { actual: [0; 32], })) } else { - Ok((Vec::new(), Lsn::ZERO)) + Ok((Vec::new(), crate::types::ReadVersions::new())) } }) }); @@ -1002,7 +1004,7 @@ mod tests { actual: [0; 32], })) } else { - Ok((Vec::new(), Lsn::ZERO)) + Ok((Vec::new(), crate::types::ReadVersions::new())) } }) }) diff --git a/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs b/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs index 748ab9430..0f4184ea6 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs @@ -178,7 +178,7 @@ impl CalvinClaim { watermark_lsn: Lsn::ZERO, error_code: Some(Box::new(ErrorCode::NotFound)), stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }; self.applied(state, reply) @@ -317,7 +317,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) diff --git a/nodedb/src/control/distributed_applier/apply_loop/write_dispatch.rs b/nodedb/src/control/distributed_applier/apply_loop/write_dispatch.rs index 967287026..564621127 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/write_dispatch.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/write_dispatch.rs @@ -303,12 +303,10 @@ fn conclude_generic_entry( // entry that failed to apply must surface to the propose waiter as a // failure, not as an empty success. let result = match submitted { - // The response carries this replica's post-write - // `coll_write_lsn` for the written collection, which the - // proposer needs as its read-your-writes floor: the version is - // minted here (the funnel's WAL append) and never travels on the - // wire, so the propose waiter is the only place it can be - // handed back. + // The response carries the versions the write stamped: the + // entry's log position on the written vShard. The proposer needs + // them as its read-your-writes floor, and the propose waiter is + // where they are handed back. Ok(resp) if resp.status == Status::Ok => Ok(AppliedWrite::from_response(&resp)), Ok(resp) => committed_response_result(&resp), Err(e) => { diff --git a/nodedb/src/control/distributed_applier/proposal_ledger.rs b/nodedb/src/control/distributed_applier/proposal_ledger.rs index 515ff9a55..5e0cb5139 100644 --- a/nodedb/src/control/distributed_applier/proposal_ledger.rs +++ b/nodedb/src/control/distributed_applier/proposal_ledger.rs @@ -131,13 +131,16 @@ impl ProposalLedger { #[cfg(test)] mod tests { use super::*; - use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; + use crate::types::{DatabaseId, TenantId, VShardId}; use crate::wal::manager::NO_APPLY_KEY; fn applied(payload: &[u8]) -> AppliedWrite { AppliedWrite { payload: payload.to_vec(), - write_version: Lsn::new(9), + write_versions: crate::types::ReadVersions::single( + VShardId::new(1), + nodedb_types::WriteVersion::logged(0, 9), + ), } } diff --git a/nodedb/src/control/distributed_applier/propose_tracker/applied_write.rs b/nodedb/src/control/distributed_applier/propose_tracker/applied_write.rs index 667d6c89d..2b59c07e8 100644 --- a/nodedb/src/control/distributed_applier/propose_tracker/applied_write.rs +++ b/nodedb/src/control/distributed_applier/propose_tracker/applied_write.rs @@ -3,53 +3,48 @@ //! What a committed entry produced, as its propose waiter receives it. use crate::bridge::envelope::Response; -use crate::types::Lsn; +use crate::types::ReadVersions; /// What a committed entry produced on the replica that applied it. /// -/// Carries the write's per-collection version alongside the payload because the -/// proposer has no other way to learn it: the version is minted inside the apply -/// path (the write funnel's WAL append), never on the wire, and the propose -/// tracker resolves on the very node that applied locally — so the version this -/// carries is that node's own, which is exactly what shard-local OCC validates -/// against. +/// Carries the write's collection versions alongside the payload because the +/// proposer has no other way to learn them: the versions are recorded inside +/// the apply path, never sent back on the wire, and the propose tracker +/// resolves on the very node that applied locally. #[derive(Debug, Clone)] pub struct AppliedWrite { /// The Data Plane's response payload, verbatim. pub payload: Vec, - /// The written collection's `coll_write_lsn` AFTER this write, in the local - /// WAL-LSN domain — the one domain OCC's read validator compares in. It is - /// NOT the Raft log index: the log index is a per-group counter that shares - /// no scale with the WAL LSNs every other feed of that map records, and - /// mixing the two silently breaks both directions of the comparison. - pub write_version: Lsn, + /// The written collection's version on the write's vShard AFTER this + /// write: the log position of the entry, the same on every replica. + pub write_versions: ReadVersions, } impl AppliedWrite { /// Take both fields off the Data Plane's response to a committed write. /// - /// `Response::read_version_lsn` is stamped by the core loop from the written - /// collection's `coll_write_lsn`, read AFTER the handler recorded this - /// write's LSN into the version index — so on a write response it is the - /// post-write version. It is `Lsn::ZERO` for a plan that maps to no single - /// user collection (see [`AppliedWrite::write_version`]). + /// `Response::read_versions` is stamped by the core loop from the written + /// collection's version, read AFTER the handler recorded this write into + /// the version index, so on a write response it is the post-write + /// version. It is empty for a plan that maps to no user collection (see + /// [`AppliedWrite::unversioned`]). pub fn from_response(response: &Response) -> Self { Self { payload: response.payload.to_vec(), - write_version: response.read_version_lsn, + write_versions: response.read_versions.clone(), } } - /// An applied entry that publishes no per-collection write-version: it wrote - /// no Data-Plane collection state (a decode skip, a forwarded read result, a - /// schema snapshot), or it was deduplicated before reaching the funnel. There - /// is no version to floor a later read at, and `Lsn::ZERO` is the read-set - /// capture's "no own-write floor" value — never a fabricated stand-in for a - /// version that exists but was not read back. + /// An applied entry that publishes no collection version: it wrote no + /// Data-Plane collection state (a decode skip, a forwarded read result, a + /// schema snapshot), or it was deduplicated before reaching the funnel. + /// There is no version to floor a later read at, and empty versions are + /// the read-set capture's "no own-write floor" value: never a fabricated + /// stand-in for a version that exists but was not read back. pub fn unversioned(payload: Vec) -> Self { Self { payload, - write_version: Lsn::ZERO, + write_versions: ReadVersions::new(), } } } diff --git a/nodedb/src/control/distributed_applier/propose_tracker/tracker.rs b/nodedb/src/control/distributed_applier/propose_tracker/tracker.rs index 7f543d0c8..1a92bfad6 100644 --- a/nodedb/src/control/distributed_applier/propose_tracker/tracker.rs +++ b/nodedb/src/control/distributed_applier/propose_tracker/tracker.rs @@ -325,28 +325,28 @@ impl ProposeTracker { mod tests { use super::*; use crate::control::distributed_applier::propose_tracker::AppliedWrite; - use crate::types::Lsn; #[test] fn propose_tracker_register_and_complete() { let tracker = ProposeTracker::new(); let mut rx = tracker.register(1, 5, 0xdead_beef); - // Waiter must receive both payload and coll_write_lsn — its only channel - // for a version minted on the apply path. + // The waiter receives the payload and the versions the apply stamped. + let version = nodedb_types::WriteVersion::logged(2, 5); + let vshard = crate::types::VShardId::new(3); assert!(tracker.complete( 1, 5, 0xdead_beef, Ok(AppliedWrite { payload: b"result".to_vec(), - write_version: Lsn::new(137), + write_versions: crate::types::ReadVersions::single(vshard, version), }), )); let result = rx.try_recv().unwrap().unwrap(); assert_eq!(result.payload, b"result"); - assert_eq!(result.write_version, Lsn::new(137)); + assert_eq!(result.write_versions.of(vshard), Some(version)); } #[test] diff --git a/nodedb/src/control/exec_receiver/backup_cut.rs b/nodedb/src/control/exec_receiver/backup_cut.rs index 49e451395..d0c157109 100644 --- a/nodedb/src/control/exec_receiver/backup_cut.rs +++ b/nodedb/src/control/exec_receiver/backup_cut.rs @@ -82,7 +82,7 @@ pub(super) async fn answer_capture_plan( crate::control::backup::cut_capture::collect::cut_and_reply(state, watermark, request) .await; Some(match answer { - Ok(payload) => ExecuteResponse::ok(vec![payload], 0, 0), + Ok(payload) => ExecuteResponse::ok(vec![payload], 0, Vec::new()), Err(error) => ExecuteResponse::err(execution_error_to_typed(error)), }) } diff --git a/nodedb/src/control/exec_receiver/executor.rs b/nodedb/src/control/exec_receiver/executor.rs index 7ede7385e..05ef7fed3 100644 --- a/nodedb/src/control/exec_receiver/executor.rs +++ b/nodedb/src/control/exec_receiver/executor.rs @@ -256,7 +256,10 @@ impl LocalPlanExecutor { { // Replicated writes carry no read watermark → 0: it floors a // session's later reads, and this RPC seam has no session. - Ok((payload, _write_version)) => ExecuteResponse::ok(vec![payload], 0, 0), + // The write's versions go back to the coordinator. + Ok((payload, write_versions)) => { + ExecuteResponse::ok(vec![payload], 0, write_versions.to_wire()) + } // A replicated write's apply verdict is a Data-Plane // verdict: carry its code, never flatten to internal. Err(e) => ExecuteResponse::err(execution_error_to_typed(e)), @@ -287,10 +290,20 @@ impl LocalPlanExecutor { ) .await { + // A read that found no row keeps its verdict and the versions it + // observed, so the coordinator answers and validates it as a + // local miss. + Ok(Ok(result)) if result.not_found => ExecuteResponse::refused_with_versions( + TypedClusterError::DataPlane { + code: crate::bridge::envelope::ErrorCode::NotFound.into(), + }, + result.watermark_lsn.as_u64(), + result.read_versions.to_wire(), + ), Ok(Ok(result)) => ExecuteResponse::ok( vec![result.payload], result.watermark_lsn.as_u64(), - result.read_version_lsn.as_u64(), + result.read_versions.to_wire(), ), Ok(Err(e)) => ExecuteResponse::err(execution_error_to_typed(e)), Err(_) => ExecuteResponse::err(TypedClusterError::DeadlineExceeded { @@ -371,7 +384,7 @@ impl LocalPlanExecutor { .send_chunk( b.payload, b.watermark_lsn.as_u64(), - b.read_version_lsn.as_u64(), + b.read_versions.to_wire(), ) .await { diff --git a/nodedb/src/control/exec_receiver/metadata_applied.rs b/nodedb/src/control/exec_receiver/metadata_applied.rs index 79b0c1d1f..56e2d99ea 100644 --- a/nodedb/src/control/exec_receiver/metadata_applied.rs +++ b/nodedb/src/control/exec_receiver/metadata_applied.rs @@ -37,7 +37,7 @@ pub(super) async fn answer_applied_plan( } }; Some(if reached { - ExecuteResponse::ok(vec![Vec::new()], 0, 0) + ExecuteResponse::ok(vec![Vec::new()], 0, Vec::new()) } else { ExecuteResponse::err(TypedClusterError::Internal { code: PLAN_DECODE_FAILED, diff --git a/nodedb/src/control/exec_receiver/stream_events.rs b/nodedb/src/control/exec_receiver/stream_events.rs index fea2f8028..0e66dab34 100644 --- a/nodedb/src/control/exec_receiver/stream_events.rs +++ b/nodedb/src/control/exec_receiver/stream_events.rs @@ -128,7 +128,7 @@ fn answer_consume_stream( ); match reply { Ok(reply) => match zerompk::to_msgpack_vec(&reply) { - Ok(payload) => ExecuteResponse::ok(vec![payload], 0, 0), + Ok(payload) => ExecuteResponse::ok(vec![payload], 0, Vec::new()), Err(error) => ExecuteResponse::err(TypedClusterError::Internal { code: PLAN_DECODE_FAILED, message: format!("CDC response encoding failed: {error}"), diff --git a/nodedb/src/control/exec_receiver/surrogate_binds.rs b/nodedb/src/control/exec_receiver/surrogate_binds.rs index f65562c54..29c75bbe4 100644 --- a/nodedb/src/control/exec_receiver/surrogate_binds.rs +++ b/nodedb/src/control/exec_receiver/surrogate_binds.rs @@ -35,7 +35,7 @@ pub(super) fn answer_binds_plan( let encoded = local_binds(state, *tenant_id, *database_id, &vshards, collections).and_then(encode_binds); Some(match encoded { - Ok(payload) => ExecuteResponse::ok(vec![payload], 0, 0), + Ok(payload) => ExecuteResponse::ok(vec![payload], 0, Vec::new()), Err(error) => ExecuteResponse::err(execution_error_to_typed(error)), }) } @@ -57,7 +57,7 @@ pub(super) fn answer_holders_plan( }; let encoded = local_holders(state, *tenant_id, *database_id, entries).and_then(encode_holders); Some(match encoded { - Ok(payload) => ExecuteResponse::ok(vec![payload], 0, 0), + Ok(payload) => ExecuteResponse::ok(vec![payload], 0, Vec::new()), Err(error) => ExecuteResponse::err(execution_error_to_typed(error)), }) } diff --git a/nodedb/src/control/exec_receiver/tenant_marks.rs b/nodedb/src/control/exec_receiver/tenant_marks.rs index 82855c4fa..e1003021c 100644 --- a/nodedb/src/control/exec_receiver/tenant_marks.rs +++ b/nodedb/src/control/exec_receiver/tenant_marks.rs @@ -63,7 +63,7 @@ pub(super) async fn answer_tenant_marks( Err(error) => return ExecuteResponse::err(execution_error_to_typed(error)), }; match encode_marks(&marks) { - Ok(payload) => ExecuteResponse::ok(vec![payload], 0, 0), + Ok(payload) => ExecuteResponse::ok(vec![payload], 0, Vec::new()), Err(error) => ExecuteResponse::err(execution_error_to_typed(error)), } } diff --git a/nodedb/src/control/gateway/core.rs b/nodedb/src/control/gateway/core.rs index 1a4913e2a..64aaca526 100644 --- a/nodedb/src/control/gateway/core.rs +++ b/nodedb/src/control/gateway/core.rs @@ -139,17 +139,16 @@ impl Gateway { /// Returns one `Vec` payload per vShard result. For point operations /// the returned Vec has exactly one element. /// - /// Thin wrapper over [`Gateway::execute_with_watermarks`] that discards the - /// per-shard read watermarks — for the ~5 existing callers that only need - /// payloads. + /// Thin wrapper over [`Gateway::execute_outcome`] that keeps only the + /// payloads — for the callers that need nothing else. pub async fn execute( &self, ctx: &QueryContext, checked: CloneCheckedTask, ) -> Result>, Error> { - self.execute_with_watermarks(ctx, checked) + self.execute_outcome(ctx, checked) .await - .map(|(payloads, _watermarks, _read_version)| payloads) + .map(|outcome| outcome.payloads) } /// Execute a trusted Control-Plane plan that has no external caller. @@ -163,38 +162,35 @@ impl Gateway { ctx: &QueryContext, plan: PhysicalPlan, ) -> Result>, Error> { - self.execute_internal_with_watermarks(ctx, plan) + self.execute_internal_outcome(ctx, plan) .await - .map(|(payloads, _watermarks, _read_version)| payloads) + .map(|outcome| outcome.payloads) } - pub(crate) async fn execute_internal_with_watermarks( + /// [`Gateway::execute_outcome`] for a trusted Control-Plane plan. + pub(crate) async fn execute_internal_outcome( &self, ctx: &QueryContext, plan: PhysicalPlan, - ) -> Result<(Vec>, Vec<(VShardId, Lsn)>, Lsn), Error> { - self.execute_plan_outcome(ctx, plan) - .await - .map(GatewayOutcome::into_parts) + ) -> Result { + self.execute_plan_outcome(ctx, plan).await } - /// Execute a pre-planned `PhysicalPlan`, returning both the raw payloads and - /// the per-shard read watermarks observed across every dispatched route. + /// Execute a pre-planned `PhysicalPlan`, returning the raw payloads, the + /// per-shard read watermarks, the read versions observed across every + /// dispatched route, and a single route's `NotFound` verdict. /// - /// Each `(vshard, watermark_lsn)` entry is one participating shard's real - /// committed LSN (local SPSC response watermark or the remote's - /// `ExecuteResponse.watermark_lsn`). The cross-node gather consumer folds - /// these into the transaction read-set so a remote-homed read records the - /// remote's actual LSN. - pub async fn execute_with_watermarks( + /// Each `(vshard, watermark_lsn)` entry is one participating shard's WAL + /// position (local SPSC response watermark or the remote's + /// `ExecuteResponse.watermark_lsn`). The read versions are data-group log + /// positions, so the transaction read-set validates them on any replica. + pub async fn execute_outcome( &self, ctx: &QueryContext, checked: CloneCheckedTask, - ) -> Result<(Vec>, Vec<(VShardId, Lsn)>, Lsn), Error> { + ) -> Result { let (plan, _lease) = authorized_plan_for_context(ctx, checked)?; - self.execute_plan_outcome(ctx, plan) - .await - .map(GatewayOutcome::into_parts) + self.execute_plan_outcome(ctx, plan).await } /// Execute an authorized plan and keep every route's result detail. @@ -260,10 +256,8 @@ impl Gateway { let max_total_bytes = shared.tuning.network.max_query_result_bytes as usize; let mut all_payloads: Vec> = Vec::new(); let mut all_shard_watermarks: Vec<(VShardId, Lsn)> = Vec::new(); - // Max-fold of the per-collection read-version across routes: a read - // targets one collection homed to one shard, so non-owning routes - // contribute `Lsn::ZERO` and the owning route's value survives. - let mut max_read_version = Lsn::ZERO; + // The read versions every route observed, one per vShard. + let mut read_versions = crate::types::ReadVersions::new(); let mut accumulated_bytes: usize = 0; for route in routes { @@ -326,9 +320,7 @@ impl Gateway { // read produces one read-set entry per shard. all_shard_watermarks.extend(outcome.shard_watermarks); not_found = single_route && outcome.not_found; - if outcome.read_version_lsn > max_read_version { - max_read_version = outcome.read_version_lsn; - } + read_versions.merge(&outcome.read_versions); for p in outcome.payloads { accumulated_bytes = accumulated_bytes.saturating_add(p.len()); @@ -355,7 +347,7 @@ impl Gateway { Ok(GatewayOutcome { payloads, shard_watermarks: all_shard_watermarks, - read_version_lsn: max_read_version, + read_versions, not_found, }) } diff --git a/nodedb/src/control/gateway/dispatch_local.rs b/nodedb/src/control/gateway/dispatch_local.rs index 4c517e668..fc26c5325 100644 --- a/nodedb/src/control/gateway/dispatch_local.rs +++ b/nodedb/src/control/gateway/dispatch_local.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use crate::Error; -use crate::bridge::envelope::{ErrorCode, PhysicalPlan, Response, Status}; +use crate::bridge::envelope::{PhysicalPlan, Response}; use crate::control::local_dispatch::reject_data_plane_error; use crate::control::server::dispatch_utils::{ AutocommitWrite, dispatch_autocommit_write, dispatch_routed_read_to_data_plane, @@ -94,7 +94,7 @@ pub(super) async fn dispatch_local( )? { let proposer = shared.async_raft_proposer()?; - let (payload, write_version) = crate::control::wal_replication::propose_replicated_entry( + let (payload, write_versions) = crate::control::wal_replication::propose_replicated_entry( shared, proposer, entry, @@ -103,10 +103,10 @@ pub(super) async fn dispatch_local( .await?; return Ok(DispatchOutcome { payloads: vec![payload], - // A write carries no read watermark (Lsn::ZERO); its post-write - // `coll_write_lsn` is surfaced via `read_version_lsn` instead. + // A write carries no read watermark (Lsn::ZERO). Its write + // versions travel in `read_versions`. shard_watermarks: vec![(vshard_id, Lsn::ZERO)], - read_version_lsn: write_version, + read_versions: write_versions, not_found: false, }); } @@ -129,8 +129,8 @@ pub(super) async fn dispatch_local( Ok(DispatchOutcome { payloads: vec![resp.payload.to_vec()], shard_watermarks: vec![(vshard_id, resp.watermark_lsn)], - read_version_lsn: resp.read_version_lsn, - not_found: is_not_found(&resp), + read_versions: resp.read_versions.clone(), + not_found: crate::control::local_dispatch::is_not_found(&resp), }) } @@ -203,11 +203,3 @@ async fn dispatch_local_plan(local: LocalPlan<'_>) -> Result { ) .await } - -/// Whether the core refused the task with `ErrorCode::NotFound`. -/// -/// `reject_data_plane_error` passes this refusal as an empty success. -/// The flag keeps the verdict for a caller that needs it. -fn is_not_found(resp: &Response) -> bool { - resp.status == Status::Error && resp.error_code.as_deref() == Some(&ErrorCode::NotFound) -} diff --git a/nodedb/src/control/gateway/dispatch_remote.rs b/nodedb/src/control/gateway/dispatch_remote.rs index 88186ff53..6ea4cea28 100644 --- a/nodedb/src/control/gateway/dispatch_remote.rs +++ b/nodedb/src/control/gateway/dispatch_remote.rs @@ -106,10 +106,10 @@ pub(super) async fn dispatch_remote( payloads: vec![resp.payload.to_vec()], shard_watermarks, // Coordinator-gathered at the exchange root: the gather folded - // the per-collection read-version across the responding shards - // and stamped it on this response, so carry it through. This - // route did not observe a version of its own to report. - read_version_lsn: resp.read_version_lsn, + // the read versions across the responding shards and stamped + // them on this response, so carry them through. This route did + // not observe a version of its own to report. + read_versions: resp.read_versions, not_found: false, }); } @@ -122,13 +122,13 @@ pub(super) async fn dispatch_remote( return Ok(DispatchOutcome { payloads: vec![merged], shard_watermarks: vec![(VShardId::new(vshard_id as u32), lsn)], - // A materialized stream reports no per-collection read-version: - // its frames carry per-batch watermarks only. `ZERO` is honest - // here rather than lossy — the streaming branch is gated on + // A materialized stream reports no read versions: its frames + // carry per-batch watermarks only. Empty is honest here rather + // than lossy: the streaming branch is gated on // `txn_id.is_none()` (`resolve/exchange.rs`), so a stream never // serves an in-transaction read and no read-set entry consumes // this value. - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), not_found: false, }); } @@ -181,23 +181,29 @@ pub(super) async fn dispatch_remote( match resp_rpc { RaftRpc::ExecuteResponse(resp) => { - if let Some(err) = resp.error { - Err(map_typed_cluster_error(err, vshard_id)) - } else { - // Key the remote's read watermark to the collection's owning - // vShard this route routed to (mirroring `dispatch_local`), NOT - // the `vshard_id % COUNT` retry hint — the read-set validator - // expects the true owning vShard as the entry key. - Ok(DispatchOutcome { - shard_watermarks: vec![( - VShardId::new(vshard_id as u32), - Lsn::new(resp.watermark_lsn), - )], - payloads: resp.payloads, - read_version_lsn: Lsn::new(resp.read_version_lsn), - not_found: false, - }) - } + // A read that found no row reads back like a local miss: an + // empty answer that keeps the `NotFound` verdict and the versions + // the read observed. Every other error crosses as its typed error. + let not_found = match resp.error { + None => false, + Some(err) => match map_typed_cluster_error(err, vshard_id) { + Error::DataPlane(crate::bridge::envelope::ErrorCode::NotFound) => true, + error => return Err(error), + }, + }; + // Key the remote's read watermark to the collection's owning + // vShard this route routed to (mirroring `dispatch_local`), NOT + // the `vshard_id % COUNT` retry hint — the read-set validator + // expects the true owning vShard as the entry key. + Ok(DispatchOutcome { + shard_watermarks: vec![( + VShardId::new(vshard_id as u32), + Lsn::new(resp.watermark_lsn), + )], + payloads: resp.payloads, + read_versions: crate::types::ReadVersions::from_wire(&resp.read_versions), + not_found, + }) } other => Err(Error::Internal { detail: format!("gateway: unexpected RPC response variant: {other:?}"), @@ -272,7 +278,7 @@ pub(super) async fn dispatch_remote_stream( let batch = RowBatch { payload: resp.payload.to_vec(), watermark_lsn: resp.watermark_lsn, - read_version_lsn: resp.read_version_lsn, + read_versions: resp.read_versions, }; return Ok(Box::pin(futures::stream::once(async move { Ok(batch) }))); } @@ -336,8 +342,8 @@ pub(super) async fn dispatch_remote_stream( payload, watermark_lsn: Lsn::new(lsn), // The `ExecuteStream` wire chunk carries only the watermark; no - // per-collection read version is threaded on this remote path. - read_version_lsn: Lsn::ZERO, + // read version is threaded on this remote path. + read_versions: crate::types::ReadVersions::new(), }, Some(Err(e)) => return Err(map_stream_cluster_error(e, vshard_id)), // Clean EOF with zero rows: a valid empty result. Return an empty stream. @@ -351,7 +357,7 @@ pub(super) async fn dispatch_remote_stream( Ok((payload, lsn)) => Ok(RowBatch { payload, watermark_lsn: Lsn::new(lsn), - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), }), Err(e) => Err(Error::Dispatch { detail: format!("remote stream terminal error: {e}"), diff --git a/nodedb/src/control/gateway/dispatcher.rs b/nodedb/src/control/gateway/dispatcher.rs index 552b01918..fecc604f3 100644 --- a/nodedb/src/control/gateway/dispatcher.rs +++ b/nodedb/src/control/gateway/dispatcher.rs @@ -12,7 +12,6 @@ use std::sync::Arc; use crate::Error; use crate::bridge::envelope::PhysicalPlan; use crate::control::server::result_stream::ResultStream; -use crate::control::server::shared::session::served_reads; use crate::control::state::SharedState; use crate::types::{DatabaseId, Lsn, TenantId, TraceId, TxnId, VShardId}; @@ -28,16 +27,14 @@ use super::version_set::GatewayVersionSet; /// /// `shard_watermarks` is one `(vshard, watermark_lsn)` per contributing shard /// — local SPSC watermark, or remote `ExecuteResponse.watermark_lsn` keyed to -/// the owning vShard — accumulated so an in-transaction read gets one -/// read-set entry per shard at its true committed LSN. +/// the owning vShard. A read-set entry takes its version from +/// `read_versions`, never from a watermark. pub struct DispatchOutcome { pub payloads: Vec>, pub shard_watermarks: Vec<(VShardId, Lsn)>, - /// This route's scanned collection's read-version LSN (`coll_write_lsn` - /// at read time), `Lsn::ZERO` for writes. Max-folded across routes — a - /// read targets one collection, so one non-zero value survives — for - /// cross-shard OCC read validation. - pub read_version_lsn: Lsn, + /// The versions this route observed, one per vShard. Folded across + /// routes for OCC read validation. + pub read_versions: crate::types::ReadVersions, /// The owning core refused the task with `ErrorCode::NotFound`. /// /// A fan-out reads it as a shard that holds no slice. A single-route @@ -79,11 +76,10 @@ pub(crate) async fn dispatch_route( } = params; reject_unadmitted_crdt_apply(&route.plan)?; let read_groups = linearizable_read_groups(shared, &route, linearizable)?; - let route_vshard = route.vshard_id; match route.decision { RouteDecision::Local => { confirm_local_read(shared, database_id, &route.plan, &read_groups, deadline_ms).await?; - let outcome = dispatch_local( + dispatch_local( route, LocalContext { shared, @@ -94,13 +90,10 @@ pub(crate) async fn dispatch_route( version_set, }, ) - .await?; - // The read's versions are this node's WAL positions. - served_reads::note(route_vshard, shared.node_id); - Ok(outcome) + .await } RouteDecision::Remote { node_id, vshard_id } => { - let outcome = dispatch_remote(RemoteDispatchArgs { + dispatch_remote(RemoteDispatchArgs { plan: route.plan, shared, node_id, @@ -114,10 +107,7 @@ pub(crate) async fn dispatch_route( linearizable, read_groups, }) - .await?; - // The read's versions are the remote node's WAL positions. - served_reads::note(route_vshard, node_id); - Ok(outcome) + .await } RouteDecision::Broadcast { .. } => { // Split into individual Local/Remote routes by the router before diff --git a/nodedb/src/control/gateway/error_map/system_dispatch_refusal.rs b/nodedb/src/control/gateway/error_map/system_dispatch_refusal.rs index 00cc42471..578828d75 100644 --- a/nodedb/src/control/gateway/error_map/system_dispatch_refusal.rs +++ b/nodedb/src/control/gateway/error_map/system_dispatch_refusal.rs @@ -86,7 +86,7 @@ async fn refuse_once( watermark_lsn: Lsn::ZERO, error_code: code.clone().map(Box::new), stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) diff --git a/nodedb/src/control/gateway/outcome.rs b/nodedb/src/control/gateway/outcome.rs index 965660db3..1b196a9ad 100644 --- a/nodedb/src/control/gateway/outcome.rs +++ b/nodedb/src/control/gateway/outcome.rs @@ -5,18 +5,18 @@ //! A transport that renders a Data-Plane `Response` gets the same shape from //! the gateway that local SPSC dispatch gives it. The shape keeps the error //! status and code of a `NotFound` verdict, the read watermark, and the -//! read-version LSN. +//! read versions. use crate::Error; use crate::bridge::envelope::{ErrorCode, Payload, Response, Status}; use crate::control::server::shared::clone_write::CloneCheckedTask; -use crate::types::{Lsn, RequestId, VShardId}; +use crate::types::{Lsn, ReadVersions, RequestId, VShardId}; use super::core::{Gateway, QueryContext, authorized_plan_for_context}; -/// A gateway execution's payloads, per-shard read watermarks, and -/// read-version LSN, as [`GatewayOutcome::into_parts`] returns them. -pub type GatewayParts = (Vec>, Vec<(VShardId, Lsn)>, Lsn); +/// A gateway execution's payloads, per-shard read watermarks, and read +/// versions, as [`GatewayOutcome::into_parts`] returns them. +pub type GatewayParts = (Vec>, Vec<(VShardId, Lsn)>, ReadVersions); /// Everything one gateway execution observed across its routes. pub struct GatewayOutcome { @@ -24,8 +24,8 @@ pub struct GatewayOutcome { pub payloads: Vec>, /// One `(vshard, watermark_lsn)` per participating shard. pub shard_watermarks: Vec<(VShardId, Lsn)>, - /// Max-folded per-collection read-version LSN. - pub read_version_lsn: Lsn, + /// The read versions every route observed, one per vShard. + pub read_versions: ReadVersions, /// A single-route plan's owning core refused it with `ErrorCode::NotFound`. /// /// Always `false` for a fan-out: there it means a shard holds no slice. @@ -33,9 +33,9 @@ pub struct GatewayOutcome { } impl GatewayOutcome { - /// Payloads, per-shard watermarks, and read-version LSN. + /// Payloads, per-shard watermarks, and read versions. pub fn into_parts(self) -> GatewayParts { - (self.payloads, self.shard_watermarks, self.read_version_lsn) + (self.payloads, self.shard_watermarks, self.read_versions) } /// The response local SPSC dispatch returns for the same task. @@ -47,7 +47,7 @@ impl GatewayOutcome { .max() .unwrap_or(Lsn::ZERO); if self.not_found { - return not_found_response(watermark_lsn, self.read_version_lsn); + return not_found_response(watermark_lsn, self.read_versions); } let payload = self .payloads @@ -55,48 +55,34 @@ impl GatewayOutcome { .next() .map(Payload::from_vec) .unwrap_or_else(Payload::empty); - response( - Status::Ok, - None, - payload, - watermark_lsn, - self.read_version_lsn, - ) + response(Status::Ok, None, payload, watermark_lsn, self.read_versions) } } impl Gateway { /// Execute one authorized task and return its Data-Plane response shape. /// - /// A `NotFound` verdict returns as an error-status response, the same as - /// local SPSC dispatch returns it. The caller records a phantom read from - /// it and renders the verdict. Every other error returns as its typed - /// `Err`. + /// A `NotFound` verdict, local or remote, returns as an error-status + /// response carrying the versions the read observed, the same as local + /// SPSC dispatch returns it. The caller records a phantom read from it + /// and renders the verdict. Every other error returns as its typed `Err`. pub async fn execute_response( &self, ctx: &QueryContext, checked: CloneCheckedTask, ) -> Result { let (plan, _lease) = authorized_plan_for_context(ctx, checked)?; - match self.execute_plan_outcome(ctx, plan).await { - Ok(outcome) => Ok(outcome.into_response()), - // A remote leaseholder returns its `NotFound` verdict as a typed - // error. It gets the same shape as a local one. - Err(Error::DataPlane(ErrorCode::NotFound)) => { - Ok(not_found_response(Lsn::ZERO, Lsn::ZERO)) - } - Err(error) => Err(error), - } + Ok(self.execute_plan_outcome(ctx, plan).await?.into_response()) } } -fn not_found_response(watermark_lsn: Lsn, read_version_lsn: Lsn) -> Response { +fn not_found_response(watermark_lsn: Lsn, read_versions: ReadVersions) -> Response { response( Status::Error, Some(ErrorCode::NotFound), Payload::empty(), watermark_lsn, - read_version_lsn, + read_versions, ) } @@ -105,7 +91,7 @@ fn response( error_code: Option, payload: Payload, watermark_lsn: Lsn, - read_version_lsn: Lsn, + read_versions: ReadVersions, ) -> Response { Response { request_id: RequestId::new(0), @@ -116,7 +102,7 @@ fn response( watermark_lsn, error_code: error_code.map(Box::new), stage_vote: None, - read_version_lsn, + read_versions, write_set: Vec::new(), } } @@ -125,6 +111,10 @@ fn response( mod tests { use super::*; + fn read_versions() -> ReadVersions { + ReadVersions::single(VShardId::new(3), nodedb_types::WriteVersion::logged(1, 5)) + } + fn outcome(not_found: bool) -> GatewayOutcome { GatewayOutcome { payloads: vec![vec![0x90]], @@ -132,7 +122,7 @@ mod tests { (VShardId::new(3), Lsn::new(7)), (VShardId::new(4), Lsn::new(9)), ], - read_version_lsn: Lsn::new(5), + read_versions: read_versions(), not_found, } } @@ -143,16 +133,16 @@ mod tests { assert_eq!(resp.status, Status::Error); assert_eq!(resp.error_code.as_deref(), Some(&ErrorCode::NotFound)); assert_eq!(resp.watermark_lsn, Lsn::new(9)); - assert_eq!(resp.read_version_lsn, Lsn::new(5)); + assert_eq!(resp.read_versions, read_versions()); } #[test] - fn a_success_keeps_its_payload_and_lsns() { + fn a_success_keeps_its_payload_watermark_and_read_versions() { let resp = outcome(false).into_response(); assert_eq!(resp.status, Status::Ok); assert!(resp.error_code.is_none()); assert_eq!(resp.payload.to_vec(), vec![0x90u8]); assert_eq!(resp.watermark_lsn, Lsn::new(9)); - assert_eq!(resp.read_version_lsn, Lsn::new(5)); + assert_eq!(resp.read_versions, read_versions()); } } diff --git a/nodedb/src/control/insert_select/orchestrator.rs b/nodedb/src/control/insert_select/orchestrator.rs index a44dd75fa..28034f6a6 100644 --- a/nodedb/src/control/insert_select/orchestrator.rs +++ b/nodedb/src/control/insert_select/orchestrator.rs @@ -223,7 +223,7 @@ pub(crate) async fn run_insert_select( watermark_lsn: max_lsn, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }) } diff --git a/nodedb/src/control/local_dispatch/collect.rs b/nodedb/src/control/local_dispatch/collect.rs index 909ec74db..28f74425a 100644 --- a/nodedb/src/control/local_dispatch/collect.rs +++ b/nodedb/src/control/local_dispatch/collect.rs @@ -170,7 +170,7 @@ mod collect_budget_tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -185,7 +185,7 @@ mod collect_budget_tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -201,7 +201,7 @@ mod collect_budget_tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -216,7 +216,7 @@ mod collect_budget_tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/local_dispatch/error_status.rs b/nodedb/src/control/local_dispatch/error_status.rs index 4cd73e121..824c21edb 100644 --- a/nodedb/src/control/local_dispatch/error_status.rs +++ b/nodedb/src/control/local_dispatch/error_status.rs @@ -8,6 +8,13 @@ use crate::bridge::envelope::{ErrorCode, Response, Status}; +/// Whether the core refused the task with `ErrorCode::NotFound`: a read +/// that found no row. [`reject_data_plane_error`] passes it as an empty +/// success, and the flag keeps the verdict for a caller that needs it. +pub(crate) fn is_not_found(resp: &Response) -> bool { + resp.status == Status::Error && resp.error_code.as_deref() == Some(&ErrorCode::NotFound) +} + /// Reject a Data-Plane error response, keeping its typed code. /// /// `NotFound` passes: it means the shard holds no slice of the collection, @@ -56,7 +63,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: Some(Box::new(code)), stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/local_dispatch/local_read.rs b/nodedb/src/control/local_dispatch/local_read.rs index ee1f232df..8c833977c 100644 --- a/nodedb/src/control/local_dispatch/local_read.rs +++ b/nodedb/src/control/local_dispatch/local_read.rs @@ -82,6 +82,7 @@ pub(crate) async fn dispatch_local_read( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }; diff --git a/nodedb/src/control/local_dispatch/mod.rs b/nodedb/src/control/local_dispatch/mod.rs index 24bd2ac16..f9219b8d4 100644 --- a/nodedb/src/control/local_dispatch/mod.rs +++ b/nodedb/src/control/local_dispatch/mod.rs @@ -11,5 +11,5 @@ mod local_read; pub(crate) use collect::{ DeadlineCollect, DispatchCollectError, collect_bounded_response, collect_under_deadline, }; -pub(crate) use error_status::reject_data_plane_error; +pub(crate) use error_status::{is_not_found, reject_data_plane_error}; pub(crate) use local_read::{LocalRead, dispatch_local_read}; diff --git a/nodedb/src/control/maintenance/clone_materializer/dispatch.rs b/nodedb/src/control/maintenance/clone_materializer/dispatch.rs index c5a4b634d..43040674f 100644 --- a/nodedb/src/control/maintenance/clone_materializer/dispatch.rs +++ b/nodedb/src/control/maintenance/clone_materializer/dispatch.rs @@ -178,6 +178,7 @@ pub(crate) async fn dispatch_on_this_node( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/control/orchestrated_write.rs b/nodedb/src/control/orchestrated_write.rs index 3cf4c82a3..dd4568dbf 100644 --- a/nodedb/src/control/orchestrated_write.rs +++ b/nodedb/src/control/orchestrated_write.rs @@ -51,17 +51,18 @@ pub(crate) async fn apply_orchestrated_write( let request_id = RequestId::new(state.request_id_counter.fetch_add(1, Ordering::Relaxed)); let deadline = crate::control::wal_replication::statement_propose_deadline(state); match propose_replicated_entry(state, proposer, entry, deadline).await { - Ok((payload, write_version)) => { + Ok((payload, write_versions)) => { let response = Response { request_id, status: Status::Ok, attempt: 1, partial: false, payload: payload.into(), - watermark_lsn: write_version, + // A write carries no read watermark. + watermark_lsn: crate::types::Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: write_version, + read_versions: write_versions, write_set: Vec::new(), }; Ok(response) @@ -82,7 +83,7 @@ fn data_plane_verdict(request_id: RequestId, code: ErrorCode) -> Response { watermark_lsn: crate::types::Lsn::ZERO, error_code: Some(Box::new(code)), stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/planner/calvin/dispatch.rs b/nodedb/src/control/planner/calvin/dispatch.rs index e5aeccc97..fc8583c3e 100644 --- a/nodedb/src/control/planner/calvin/dispatch.rs +++ b/nodedb/src/control/planner/calvin/dispatch.rs @@ -214,7 +214,7 @@ mod tests { use crate::control::server::shared::session::read_set::{ EngineTag, ReadKey, ReadOrigin, ReadSetEntry, }; - use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; + use crate::types::{DatabaseId, TenantId, VShardId}; use nodedb_physical::physical_plan::{ColumnarOp, CrdtOp, DocumentOp, PhysicalPlan}; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; use nodedb_types::QualifiedCollection; @@ -489,13 +489,11 @@ mod tests { // them (`session::commit::run_commit`). // // WHY this must stay `MultiShard`: only the `MultiShard` branch of COMMIT - // commits through the Calvin barrier (`run_commit_calvin`), which validates - // B's read slice on B's OWNING node using the real per-shard `read_lsn`. If a - // foreign read failed to widen the class, COMMIT will take the `SingleShard` - // branch and run only the local-WAL `si_conflict_abort`, which never sees a - // stale read on the remote owner — silently committing a non-serializable - // cross-node transaction. This test guarantees a future refactor of - // `read_vshards_of` / `classify_dispatch` cannot reopen that hole. + // commits through the Calvin barrier (`run_commit_calvin`), which + // validates B's read slice on B's participants inside the barrier, in + // the same order as every write to B. This test guarantees a future + // refactor of `read_vshards_of` / `classify_dispatch` keeps a foreign + // read inside the barrier. let (write_coll, read_coll) = two_distinct_vshard_collections(); let write_vshard = CollectionKey::from_bare(DatabaseId::DEFAULT, &write_coll) .vshard() @@ -512,11 +510,9 @@ mod tests { tenant_id: TenantId::new(1), collection: read_coll.clone(), key: ReadKey::Predicate, - read_lsn: Lsn::new(1), - read_version_lsn: Lsn::ZERO, + read_version: nodedb_types::WriteVersion::logged(0, 1), origin: ReadOrigin::Session, home: None, - home_node: 0, }; // The homing step under test: a foreign-collection read must home to a @@ -538,8 +534,8 @@ mod tests { } other => panic!( "expected MultiShard for write-on-{write_vshard} + foreign-read-on-{read_vshard}, \ - got {other:?} (a SingleShard here would route COMMIT to local-WAL \ - si_conflict_abort and reopen the cross-node serializability hole)" + got {other:?} (a SingleShard here validates the read outside the \ + Calvin barrier)" ), } } diff --git a/nodedb/src/control/planner/calvin/dispatch_multi.rs b/nodedb/src/control/planner/calvin/dispatch_multi.rs index f9e9a2337..166e846ba 100644 --- a/nodedb/src/control/planner/calvin/dispatch_multi.rs +++ b/nodedb/src/control/planner/calvin/dispatch_multi.rs @@ -208,7 +208,7 @@ pub(crate) async fn dispatch_tasks_to_calvin( mod tests { use super::*; use crate::control::server::shared::session::read_set::{EngineTag, ReadKey, ReadOrigin}; - use crate::types::{DatabaseId, KeyRepr, Lsn, VShardId}; + use crate::types::{DatabaseId, KeyRepr, VShardId}; use nodedb_cluster::calvin::types::TxnIdWire; use nodedb_physical::physical_plan::{DocumentOp, GraphOp, PhysicalPlan}; use nodedb_physical::physical_task::PostSetOp; @@ -244,11 +244,9 @@ mod tests { key: ReadKey::Point { repr: KeyRepr::Surrogate(1), }, - read_lsn: Lsn::new(1), - read_version_lsn: Lsn::new(1), + read_version: nodedb_types::WriteVersion::logged(0, 1), origin: ReadOrigin::Session, home: None, - home_node: 0, } } diff --git a/nodedb/src/control/planner/calvin/submit/local.rs b/nodedb/src/control/planner/calvin/submit/local.rs index ead318c00..51ffc4e9a 100644 --- a/nodedb/src/control/planner/calvin/submit/local.rs +++ b/nodedb/src/control/planner/calvin/submit/local.rs @@ -39,7 +39,7 @@ pub(super) fn synthetic_returning_response(payload_bytes: Vec) -> Response { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/planner/calvin/tx_class/dependent_builder.rs b/nodedb/src/control/planner/calvin/tx_class/dependent_builder.rs index 4b473fba6..87200af9e 100644 --- a/nodedb/src/control/planner/calvin/tx_class/dependent_builder.rs +++ b/nodedb/src/control/planner/calvin/tx_class/dependent_builder.rs @@ -39,7 +39,7 @@ use crate::control::planner::calvin::write_class::is_write_plan; /// /// `reads` is the neutral session read-set, projected onto the `TxClass`'s /// routing/identity `read_set` (collection-homed) so read shards are enumerated -/// as participants, and onto `versioned_reads` (LSN-versioned OCC validation +/// as participants, and onto `versioned_reads` (write-versioned OCC validation /// set) for commit-time optimistic-concurrency validation. Autocommit paths /// pass an empty slice. /// @@ -145,7 +145,7 @@ fn build_dependent_tx_class_impl( detail: format!("failed to encode PhysicalPlan vec for Calvin dependent TxClass: {e}"), })?; - // versioned_reads carries the LSN-versioned OCC validation set, populated + // versioned_reads carries the write-versioned OCC validation set, populated // from the same session read-set the routing `read_set` above was built // from. let versioned_reads = versioned_reads_from(reads); diff --git a/nodedb/src/control/planner/calvin/tx_class/shared.rs b/nodedb/src/control/planner/calvin/tx_class/shared.rs index 578c4d838..5cdd4efa4 100644 --- a/nodedb/src/control/planner/calvin/tx_class/shared.rs +++ b/nodedb/src/control/planner/calvin/tx_class/shared.rs @@ -13,14 +13,13 @@ use nodedb_cluster::calvin::types::{ }; use nodedb_types::KeyRepr; -/// Map the neutral session read-set into the replicated, LSN-versioned +/// Map the neutral session read-set into the replicated, versioned /// [`VersionedReadSet`] carried on the `TxClass`. /// /// Each [`ReadSetEntry`] becomes one [`VersionedReadEntry`], preserving -/// engine, collection, `read_version_lsn` (the collection's write floor — -/// the sound OCC comparand, not the core-global `read_lsn`), and the -/// point/predicate distinction. Own-overlay exclusion already happened at -/// capture time, so this is a faithful 1:1 projection. +/// engine, collection, read version, home, and the point/predicate +/// distinction. Own-overlay exclusion already happened at capture time, so +/// this is a faithful 1:1 projection. pub(super) fn versioned_reads_from(reads: &[ReadSetEntry]) -> VersionedReadSet { VersionedReadSet::new( reads @@ -41,9 +40,8 @@ pub(super) fn versioned_reads_from(reads: &[ReadSetEntry]) -> VersionedReadSet { hi: hi.clone(), }, }, - read_lsn: entry.read_version_lsn, + read_version: entry.read_version, home_vshard: entry.home.map(|home| home.as_u32()), - served_by: entry.home_node, }) .collect(), ) @@ -51,7 +49,7 @@ pub(super) fn versioned_reads_from(reads: &[ReadSetEntry]) -> VersionedReadSet { /// Build the routing and lock `read_set` for a Calvin `TxClass` from the /// neutral session read-set. This is the key-IDENTITY set for participants -/// and locks, not the LSN-versioned OCC set ([`versioned_reads_from`]). +/// and locks, not the versioned OCC set ([`versioned_reads_from`]). /// /// Each read maps to the keys the scheduler locks `Shared`: /// - A point read locks its row: a surrogate, a KV key, or an edge's two diff --git a/nodedb/src/control/planner/calvin/tx_class/static_builder.rs b/nodedb/src/control/planner/calvin/tx_class/static_builder.rs index c7f4c5b73..00178eb3f 100644 --- a/nodedb/src/control/planner/calvin/tx_class/static_builder.rs +++ b/nodedb/src/control/planner/calvin/tx_class/static_builder.rs @@ -26,7 +26,7 @@ use super::write_keys::task_write_keys; /// `reads` is the neutral session read-set captured during the transaction; it /// is projected onto the `TxClass`'s routing/identity `read_set` (collection- /// homed) so a txn that writes shard A but reads shard B enumerates B as a -/// participant, and onto `versioned_reads` (LSN-versioned OCC validation set) +/// participant, and onto `versioned_reads` (write-versioned OCC validation set) /// for commit-time optimistic-concurrency validation. Autocommit and /// pure-write paths pass an empty slice, yielding an empty read_set and /// versioned_reads. @@ -102,8 +102,8 @@ fn build_static_tx_class_impl( // must NOT enter the OCC read set. The txn's own staged write advances that // collection's write floor, so validating the earlier read against it will // flag it stale and abort the commit — a false serialization conflict. This - // mirrors the written-collection exclusion the single-shard - // `si_conflict_abort` path already applies. + // mirrors the written-collection exclusion the non-Calvin commit check + // (`session::commit::read_validation`) applies. // // A PLAN-DERIVATION read is exempt, and the exemption is what makes the // derivation guard exist at all. Such a read observed COMMITTED base state @@ -144,7 +144,7 @@ fn build_static_tx_class_impl( detail: format!("failed to encode PhysicalPlan vec for Calvin TxClass: {e}"), })?; - // versioned_reads carries the LSN-versioned OCC validation set, populated + // versioned_reads carries the write-versioned OCC validation set, populated // from the same session read-set the routing `read_set` above was built // from. let versioned_reads = versioned_reads_from(&owned_reads); @@ -179,7 +179,7 @@ fn build_static_tx_class_impl( mod tests { use super::*; use crate::control::server::shared::session::read_set::{EngineTag, ReadKey, ReadOrigin}; - use crate::types::{DatabaseId, KeyRepr, Lsn, VShardId}; + use crate::types::{DatabaseId, KeyRepr, VShardId}; use nodedb_physical::physical_plan::DocumentOp; use nodedb_types::Surrogate; @@ -222,14 +222,14 @@ mod tests { } } - fn read_entry(collection: &str, key: ReadKey, read_lsn: u64) -> ReadSetEntry { - origin_read_entry(collection, key, read_lsn, ReadOrigin::Session) + fn read_entry(collection: &str, key: ReadKey, read_index: u64) -> ReadSetEntry { + origin_read_entry(collection, key, read_index, ReadOrigin::Session) } fn origin_read_entry( collection: &str, key: ReadKey, - read_lsn: u64, + read_index: u64, origin: ReadOrigin, ) -> ReadSetEntry { ReadSetEntry { @@ -238,13 +238,9 @@ mod tests { tenant_id: TenantId::new(1), collection: collection.to_owned(), key, - read_lsn: Lsn::new(read_lsn), - // The per-collection read-version is the OCC comparand - // `versioned_reads_from` propagates; give it the same synthetic LSN. - read_version_lsn: Lsn::new(read_lsn), + read_version: nodedb_types::WriteVersion::logged(0, read_index), origin, home: None, - home_node: 0, } } @@ -274,8 +270,8 @@ mod tests { let (col_a, col_b) = two_distinct_collections(); let tasks = vec![point_insert_task(&col_a, 1), point_insert_task(&col_b, 2)]; - // Synthetic read-set: one point read (surrogate identity) at LSN 7 and - // one collection-scoped predicate read at LSN 11, on two collections + // Synthetic read-set: one point read (surrogate identity) at index 7 and + // one collection-scoped predicate read at index 11, on two collections // distinct from the write collections. let reads = vec![ read_entry( @@ -291,7 +287,7 @@ mod tests { let tx = build_static_tx_class(&tasks, TenantId::new(1), &reads) .expect("valid multi-vShard TxClass"); - // versioned_reads (the LSN-versioned OCC validation set) carries the + // versioned_reads (the versioned OCC validation set) carries the // same session reads, 1:1, for commit-time validation. assert_eq!( tx.versioned_reads.len(), @@ -300,7 +296,7 @@ mod tests { ); for (entry, read) in tx.versioned_reads.iter().zip(reads.iter()) { assert_eq!(entry.collection, read.collection); - assert_eq!(entry.read_lsn, read.read_version_lsn); + assert_eq!(entry.read_version, read.read_version); let expected_key = match &read.key { ReadKey::Point { repr } => { nodedb_cluster::calvin::types::ReadKeyIdent::Point(repr.clone()) @@ -463,7 +459,7 @@ mod tests { /// A PLAN-DERIVATION read of a collection the transaction WRITES survives /// the own-write exclusion, reaching BOTH the routing read_set and the - /// LSN-versioned `versioned_reads` the Data Plane's + /// write-versioned `versioned_reads` the Data Plane's /// `read_set_still_current` check consumes. /// /// This is the whole of the cross-shard materialized-sum guard: the delta @@ -494,7 +490,10 @@ mod tests { "a derivation read must reach versioned_reads — that set IS the guard" ); assert_eq!(tx.versioned_reads.0[0].collection, col_a); - assert_eq!(tx.versioned_reads.0[0].read_lsn, Lsn::new(42)); + assert_eq!( + tx.versioned_reads.0[0].read_version, + nodedb_types::WriteVersion::logged(0, 42) + ); assert_eq!( tx.versioned_reads.0[0].key, nodedb_cluster::calvin::types::ReadKeyIdent::Point(KeyRepr::Surrogate(11)) diff --git a/nodedb/src/control/planner/materialized_sum/predicate.rs b/nodedb/src/control/planner/materialized_sum/predicate.rs index 858847451..979077aa3 100644 --- a/nodedb/src/control/planner/materialized_sum/predicate.rs +++ b/nodedb/src/control/planner/materialized_sum/predicate.rs @@ -114,8 +114,7 @@ pub(super) async fn resolve_predicate_sum_targets( // observation this settlement rests on is the whole collection: a row // that JOINS the match set after the scan has to invalidate it too. source_row: None, - read_version_lsn: read.read_version_lsn, - served_by: read.served_by, + read_version: read.read_version, }; let settlement = settle_cross_shard_images(&bindings, &input, &resolved, txn_id, tenant_id, database_id)?; diff --git a/nodedb/src/control/planner/materialized_sum/recon.rs b/nodedb/src/control/planner/materialized_sum/recon.rs index 8fdb0e0fc..51b6dc170 100644 --- a/nodedb/src/control/planner/materialized_sum/recon.rs +++ b/nodedb/src/control/planner/materialized_sum/recon.rs @@ -39,7 +39,7 @@ use nodedb_types::{Surrogate, TenantId}; use crate::control::state::SharedState; -use crate::types::{DatabaseId, Lsn, TraceId, TxnId}; +use crate::types::{DatabaseId, TraceId, TxnId}; use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan}; /// What a plan-time reconnaissance read observed, and the version it observed @@ -53,13 +53,10 @@ use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan}; pub(crate) struct ReconRead { /// The decoded rows. pub rows: T, - /// The source collection's write floor at read time — the comparand - /// cross-shard OCC validation checks the read against. - pub read_version_lsn: Lsn, - /// The node that served the read. `read_version_lsn` is a position in - /// that node's WAL, so the commit vote compares it only on that node. - /// `0` when no one node is known to have served it. - pub served_by: u64, + /// The source collection's version on its vShard at read time — the + /// comparand OCC validation checks the read against. Every replica + /// numbers versions alike, so any replica can have served the read. + pub read_version: nodedb_types::WriteVersion, } /// Scan `collection` for the rows `filters` matches, returning each row's full @@ -106,8 +103,7 @@ pub(in crate::control::planner) async fn recon_scan_rows( } Ok(ReconRead { rows, - read_version_lsn: read.read_version_lsn, - served_by: read.served_by, + read_version: read.read_version, }) } @@ -161,8 +157,7 @@ pub(crate) async fn recon_point_row( .iter() .find(|payload| !payload.is_empty()) .and_then(|payload| nodedb_types::json_from_msgpack(payload.as_slice()).ok()), - read_version_lsn: read.read_version_lsn, - served_by: read.served_by, + read_version: read.read_version, }) } @@ -173,14 +168,15 @@ pub(crate) async fn recon_point_row( /// write with no target to address — so every plan-time read routes through /// the gateway. /// -/// The gateway notes the node that served each vShard it read. The read -/// validates on `collection`'s vShard, so that vShard's note names the node -/// whose WAL numbers `read_version_lsn`. +/// The read validates on `collection`'s vShard, so its version is the one +/// the read reported for that vShard. A vShard the read reported no version +/// of had no write the serving core knew of, so the read observed it at +/// `WriteVersion::ZERO`. /// -/// `read_version_lsn` is the collection's COMMITTED write floor whether or not -/// `txn_id` is set: a staged write appends no WAL record and moves no write -/// version. So an entry stamped with it still detects a concurrent commit to -/// the rows read, and never this transaction's own staged writes. +/// The version is the collection's COMMITTED version whether or not `txn_id` +/// is set: a staged write moves no write version. So an entry stamped with it +/// still detects a concurrent commit to the rows read, and never this +/// transaction's own staged writes. async fn execute_read( state: &SharedState, tenant_id: TenantId, @@ -190,9 +186,7 @@ async fn execute_read( plan: PhysicalPlan, ) -> crate::Result>>> { let validation_vshard = - nodedb_types::CollectionKey::from_qualified_str(database_id, collection)? - .vshard() - .as_u32(); + nodedb_types::CollectionKey::from_qualified_str(database_id, collection)?.vshard(); let gateway = state.installed_gateway()?; let gw_ctx = crate::control::gateway::core::QueryContext { tenant_id, @@ -204,16 +198,13 @@ async fn execute_read( linearizable: true, }; // A shard verdict keeps its own typed error. - let (payloads, _watermarks, read_version_lsn) = gateway - .execute_internal_with_watermarks(&gw_ctx, plan) - .await?; + let outcome = gateway.execute_internal_outcome(&gw_ctx, plan).await?; Ok(ReconRead { - rows: payloads, - read_version_lsn, - served_by: crate::control::server::shared::session::read_set::serving_node( - state, - validation_vshard, - ), + read_version: outcome + .read_versions + .of(validation_vshard) + .unwrap_or_default(), + rows: outcome.payloads, }) } diff --git a/nodedb/src/control/planner/materialized_sum/resolve.rs b/nodedb/src/control/planner/materialized_sum/resolve.rs index e4c7c7ec3..2ed6bb5ee 100644 --- a/nodedb/src/control/planner/materialized_sum/resolve.rs +++ b/nodedb/src/control/planner/materialized_sum/resolve.rs @@ -143,8 +143,7 @@ pub async fn resolve_materialized_sum_targets( source_collection: collection, images: &images.images, source_row: Some(scope.surrogate), - read_version_lsn: images.read_version_lsn, - served_by: images.served_by, + read_version: images.read_version, }; let settlement = settle_cross_shard_images( &bindings, diff --git a/nodedb/src/control/planner/materialized_sum/settle.rs b/nodedb/src/control/planner/materialized_sum/settle.rs index 8eeffb5bc..6ecb7d014 100644 --- a/nodedb/src/control/planner/materialized_sum/settle.rs +++ b/nodedb/src/control/planner/materialized_sum/settle.rs @@ -75,7 +75,7 @@ use crate::control::server::shared::session::read_set::{ }; use crate::engine::document::store::StorageKey; use crate::query::{sum_target_is_co_resident, sum_target_vshard}; -use crate::types::{DatabaseId, KeyRepr, Lsn, TenantId}; +use crate::types::{DatabaseId, KeyRepr, TenantId}; /// One source write's plan-time images, ready to settle. pub(super) struct SettleInput<'a> { @@ -91,11 +91,8 @@ pub(super) struct SettleInput<'a> { /// that row. `None` for a predicate-shaped write, whose observation is the /// whole collection and whose entry is collection-scoped accordingly. pub source_row: Option, - /// Version the images were read at. - pub read_version_lsn: Lsn, - /// The node that served the image read. `read_version_lsn` is a position - /// in its WAL, so only that node's commit vote can find the read current. - pub served_by: u64, + /// Version the images were read at, on the source collection's vShard. + pub read_version: nodedb_types::WriteVersion, } /// What settling produced. @@ -274,8 +271,7 @@ fn image_read_entry( }, None => ReadKey::Predicate, }, - read_lsn: input.read_version_lsn, - read_version_lsn: input.read_version_lsn, + read_version: input.read_version, // A DERIVATION read, never a read-your-own-write. The delta shipped to // the target rests entirely on the image. The statement writes the // source collection too, so an entry marked `Session` here will be @@ -283,7 +279,7 @@ fn image_read_entry( // validated against a concurrent writer. // // Inside a transaction the image is read through its staging overlay. - // `read_version_lsn` is still the committed write floor, because a + // `read_version` is still the committed version, because a // staged write moves no write version. So the entry aborts the commit // when another transaction commits a write to the row first. It never // aborts on this transaction's own staged writes, which the image @@ -292,7 +288,6 @@ fn image_read_entry( // surrogate. A concurrent commit of that row invalidates it. origin: ReadOrigin::PlanDerivation, home: None, - home_node: input.served_by, } } @@ -407,8 +402,7 @@ mod tests { source_collection: &source, images: &images, source_row: Some(Surrogate::new(11)), - read_version_lsn: Lsn::new(42), - served_by: 7, + read_version: nodedb_types::WriteVersion::logged(3, 42), }; let settlement = settle_cross_shard_images( &[binding(&target)], @@ -443,8 +437,7 @@ mod tests { source_collection: &source, images: &images, source_row: Some(Surrogate::new(11)), - read_version_lsn: Lsn::new(42), - served_by: 7, + read_version: nodedb_types::WriteVersion::logged(3, 42), }; let settlement = settle_cross_shard_images( &[binding(&target)], @@ -484,8 +477,7 @@ mod tests { source_collection: &source, images: &images, source_row: Some(Surrogate::new(11)), - read_version_lsn: Lsn::new(42), - served_by: 7, + read_version: nodedb_types::WriteVersion::logged(3, 42), }; let settlement = settle_cross_shard_images( &[binding(&target)], @@ -509,8 +501,7 @@ mod tests { source_collection: &source, images: &images, source_row: Some(Surrogate::new(11)), - read_version_lsn: Lsn::new(42), - served_by: 7, + read_version: nodedb_types::WriteVersion::logged(3, 42), }; let settlement = settle_cross_shard_images( &[binding(&target)], @@ -538,8 +529,7 @@ mod tests { source_collection: &source, images: &images, source_row: Some(Surrogate::new(11)), - read_version_lsn: Lsn::new(42), - served_by: 7, + read_version: nodedb_types::WriteVersion::logged(3, 42), }; // A binding whose target IS the source collection is co-resident by // construction, whatever the hash function does. @@ -568,8 +558,7 @@ mod tests { source_collection: &source, images: &images, source_row: Some(Surrogate::new(11)), - read_version_lsn: Lsn::new(42), - served_by: 7, + read_version: nodedb_types::WriteVersion::logged(3, 42), }; let settlement = settle_cross_shard_images( &[binding(&target)], @@ -588,11 +577,10 @@ mod tests { repr: KeyRepr::Surrogate(11) } ); - assert_eq!(settlement.reads[0].read_version_lsn, Lsn::new(42)); assert_eq!( - settlement.reads[0].home_node, 7, - "the entry names the node that served the image read; `0` makes \ - every commit vote treat the read as changed" + settlement.reads[0].read_version, + nodedb_types::WriteVersion::logged(3, 42), + "the entry carries the version the image read observed" ); } @@ -607,8 +595,7 @@ mod tests { source_collection: &source, images: &images, source_row: None, - read_version_lsn: Lsn::new(42), - served_by: 7, + read_version: nodedb_types::WriteVersion::logged(3, 42), }; let settlement = settle_cross_shard_images( &[binding(&target)], diff --git a/nodedb/src/control/planner/materialized_sum/stored.rs b/nodedb/src/control/planner/materialized_sum/stored.rs index 0fa8a4125..142566460 100644 --- a/nodedb/src/control/planner/materialized_sum/stored.rs +++ b/nodedb/src/control/planner/materialized_sum/stored.rs @@ -30,7 +30,7 @@ use super::recon::recon_point_row; use super::resolve::lookup_join_value; use super::resolve_target::ResolvedTargets; use crate::control::state::SharedState; -use crate::types::{DatabaseId, Lsn, TenantId, TraceId}; +use crate::types::{DatabaseId, TenantId, TraceId}; /// How a point-shaped write's POST-image is formed. /// @@ -82,10 +82,8 @@ pub(super) struct StoredRowScope<'a> { pub(super) struct StoredImages { /// One pair per row this write touches — at most one, for a point shape. pub images: Vec<(Option, Option)>, - /// The source collection's write floor at read time. - pub read_version_lsn: Lsn, - /// The node that served the read, whose WAL numbers `read_version_lsn`. - pub served_by: u64, + /// The source collection's version on its vShard at read time. + pub read_version: nodedb_types::WriteVersion, } /// The stored row an op rewrites or removes, or `None` for every op that @@ -225,8 +223,7 @@ pub(super) async fn extend_with_stored_row( let outcome = StoredImages { images: images_of(scope, read.rows.as_ref())?, - read_version_lsn: read.read_version_lsn, - served_by: read.served_by, + read_version: read.read_version, }; if read.rows.is_none() { diff --git a/nodedb/src/control/planner/procedural/executor/core/dispatch.rs b/nodedb/src/control/planner/procedural/executor/core/dispatch.rs index 72cb603b6..f11e5325a 100644 --- a/nodedb/src/control/planner/procedural/executor/core/dispatch.rs +++ b/nodedb/src/control/planner/procedural/executor/core/dispatch.rs @@ -513,7 +513,7 @@ mod origination_tests { watermark_lsn: Lsn::ZERO, error_code, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }; side.response_tx diff --git a/nodedb/src/control/planner/sql_plan_convert/array_convert/dml.rs b/nodedb/src/control/planner/sql_plan_convert/array_convert/dml.rs index 46bf76048..6e1538742 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_convert/dml.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_convert/dml.rs @@ -2,8 +2,8 @@ //! `INSERT INTO ARRAY` / `DELETE FROM ARRAY` lowering to `PhysicalTask`. -use nodedb_array::coord::encode::encode_hilbert_prefix; use nodedb_array::schema::ArraySchema; +use nodedb_array::tile::cell_tile_prefix; use nodedb_array::types::ArrayId; use nodedb_sql::types_array::{ArrayCoordLiteral, ArrayInsertRow}; @@ -84,8 +84,10 @@ pub(in super::super) fn convert_insert_array( detail: format!("array coord pk encode: {e}"), })?; let surrogate = ctx.surrogate_for_pk(ctx.collection_key(name), &pk_bytes)?; + // A cell routes by its tile's key, the key a slice's shard + // fan-out and each shard's tile filter use. let hilbert = - encode_hilbert_prefix(&schema, &coord).map_err(|e| crate::Error::PlanError { + cell_tile_prefix(&schema, &coord).map_err(|e| crate::Error::PlanError { detail: format!("INSERT INTO ARRAY {name}: Hilbert prefix: {e}"), })?; let cell = ArrayPutCell { @@ -211,7 +213,7 @@ pub(in super::super) fn convert_delete_array( for row in coords { let typed = coerce_coords(row, &schema)?; let hilbert = - encode_hilbert_prefix(&schema, &typed).map_err(|e| crate::Error::PlanError { + cell_tile_prefix(&schema, &typed).map_err(|e| crate::Error::PlanError { detail: format!("DELETE FROM ARRAY {name}: Hilbert prefix: {e}"), })?; let cell = ArrayDeleteCell { diff --git a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs index 7daa52d12..453669ffc 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs @@ -4,9 +4,10 @@ use nodedb_array::query::slice::{DimRange, Slice}; use nodedb_array::schema::ArraySchema; -use nodedb_array::tile::layout::tile_id_for_cell; +use nodedb_array::schema::DimSpec; +use nodedb_array::tile::{tile_prefix, tiles_per_dim}; use nodedb_array::types::ArrayId; -use nodedb_array::types::coord::value::CoordValue; +use nodedb_array::types::domain::DomainBound; use nodedb_sql::temporal::TemporalScope; use nodedb_sql::types_array::ArraySliceAst; @@ -66,10 +67,8 @@ pub(crate) fn convert_slice( // intercepts it and calls ArrayCoordinator::coord_slice instead // of sending to the local SPSC bridge. // - // Compute a conservative Hilbert bounding range from the slice's - // dim-range corners so the coordinator only fans out to the shards - // whose Hilbert buckets overlap the query range, rather than all - // 1024 shards (the "empty ranges = unbounded" fallback). + // The coordinator fans out only to the shards that own a tile the + // slice covers. An empty list fans out to every shard. let slice_hilbert_ranges = compute_slice_hilbert_ranges(&schema, &slice.dim_ranges, entry.prefix_bits); PhysicalPlan::ClusterArray(ClusterArrayOp::Slice { @@ -105,92 +104,94 @@ pub(crate) fn convert_slice( }]) } -/// Compute a conservative Hilbert bounding range for a slice predicate. -/// -/// For each constrained dimension, uses the lo and hi bounds as the range -/// endpoints. Generates all 2^num_dims corner coordinates of the bounding -/// box and computes the Hilbert prefix for each. Returns a single -/// `(min_prefix, max_prefix)` range that conservatively covers all cells -/// that can match the slice. +/// The most tiles a slice's shard fan-out enumerates. A slice covering more +/// tiles fans out to every shard. +const MAX_FAN_OUT_TILES: u128 = 4096; + +/// The Hilbert ranges a slice's shard fan-out covers: one `(key, key)` pair +/// per routing bucket, each holding the key of one tile in that bucket. /// -/// Falls back to an empty vec (unbounded = contact all shards) if the -/// schema has more than 16 dimensions (2^16 corners will be excessive) -/// or if any Hilbert encoding fails. +/// Every tile whose cells can match the slice is enumerated, so every shard +/// that holds such a cell, or will hold one a later write lands there, is +/// covered. A transaction's read of the slice then validates on each of +/// those shards. A cell routes by its tile's key (`cell_tile_prefix`), the +/// same key enumerated here. /// -/// The result can over-estimate the shard set — the shard-side slice -/// filter is always applied and never produces false positives. +/// Returns an empty vec, which fans out to every shard, when the slice +/// covers more than [`MAX_FAN_OUT_TILES`] tiles, when the schema has no +/// dimension, or when a tile key fails to encode. fn compute_slice_hilbert_ranges( schema: &ArraySchema, dim_ranges: &[Option], - _prefix_bits: u8, + prefix_bits: u8, ) -> Vec<(u64, u64)> { - use nodedb_array::types::domain::DomainBound; - - let ndim = schema.dims.len(); - if ndim == 0 || ndim > 16 { - return Vec::new(); // fallback: unbounded + if schema.dims.is_empty() || prefix_bits == 0 || prefix_bits > 16 { + return Vec::new(); } - - // Build lo/hi bound for each dim. Unconstrained dims use the schema domain. - let mut lo_vals: Vec = Vec::with_capacity(ndim); - let mut hi_vals: Vec = Vec::with_capacity(ndim); - - for (i, dr_opt) in dim_ranges.iter().enumerate() { - let dim = &schema.dims[i]; - let (lo_bound, hi_bound) = if let Some(dr) = dr_opt { - (dr.lo.clone(), dr.hi.clone()) - } else { - (dim.domain.lo.clone(), dim.domain.hi.clone()) - }; - - let lo_cv = match lo_bound { - DomainBound::Int64(v) => CoordValue::Int64(v), - DomainBound::Float64(v) => CoordValue::Float64(v), - DomainBound::TimestampMs(v) => CoordValue::TimestampMs(v), - DomainBound::String(v) => CoordValue::String(v), - }; - let hi_cv = match hi_bound { - DomainBound::Int64(v) => CoordValue::Int64(v), - DomainBound::Float64(v) => CoordValue::Float64(v), - DomainBound::TimestampMs(v) => CoordValue::TimestampMs(v), - DomainBound::String(v) => CoordValue::String(v), - }; - - lo_vals.push(lo_cv); - hi_vals.push(hi_cv); + let spans: Vec<(u64, u64)> = schema + .dims + .iter() + .zip(schema.tile_extents.iter()) + .enumerate() + .map(|(i, (dim, extent))| { + dim_tile_span(dim, *extent, dim_ranges.get(i).and_then(Option::as_ref)) + }) + .collect(); + let tiles: u128 = spans + .iter() + .map(|(lo, hi)| u128::from(hi - lo) + 1) + .fold(1u128, |acc, n| acc.saturating_mul(n)); + if tiles > MAX_FAN_OUT_TILES { + return Vec::new(); } - // Generate all 2^ndim corners and compute their Hilbert prefix. - let num_corners = 1usize << ndim; - let mut min_prefix = u64::MAX; - let mut max_prefix = 0u64; - let mut found_any = false; - - for mask in 0..num_corners { - let corner: Vec = (0..ndim) - .map(|i| { - if (mask >> i) & 1 == 1 { - hi_vals[i].clone() - } else { - lo_vals[i].clone() - } - }) - .collect(); - - // Use the tile prefix (computed from tile indices = coord / tile_extents), - // not the cell prefix — tiles store and the per-shard filter compares - // against tile-level Hilbert keys. - if let Ok(tile_id) = tile_id_for_cell(schema, &corner, 0) { - min_prefix = min_prefix.min(tile_id.hilbert_prefix); - max_prefix = max_prefix.max(tile_id.hilbert_prefix); - found_any = true; + let shift = 64 - u32::from(prefix_bits); + let mut buckets: std::collections::BTreeMap = std::collections::BTreeMap::new(); + let mut indices: Vec = spans.iter().map(|(lo, _)| *lo).collect(); + loop { + let Ok(key) = tile_prefix(schema, &indices) else { + return Vec::new(); + }; + buckets.entry(key >> shift).or_insert(key); + // Advance the odometer over the per-dimension tile spans. + let mut dim = 0; + loop { + if dim == indices.len() { + return buckets.into_values().map(|key| (key, key)).collect(); + } + if indices[dim] < spans[dim].1 { + indices[dim] += 1; + break; + } + indices[dim] = spans[dim].0; + dim += 1; } } +} - if found_any { - vec![(min_prefix, max_prefix)] - } else { - Vec::new() // fallback: unbounded +/// The inclusive tile index span along `dim` that `range` covers, the whole +/// dimension when `range` is `None`. A string dimension tiles by hash, so a +/// range on it covers every tile. A bound outside the domain is clamped to +/// the domain's tiles. +fn dim_tile_span(dim: &DimSpec, extent: u64, range: Option<&DimRange>) -> (u64, u64) { + let last = u64::try_from(tiles_per_dim(dim, extent).saturating_sub(1)).unwrap_or(u64::MAX); + let extent = u128::from(extent.max(1)); + let index = |bound: &DomainBound| -> Option { + let offset: u128 = match (bound, &dim.domain.lo) { + (DomainBound::Int64(v), DomainBound::Int64(lo)) + | (DomainBound::TimestampMs(v), DomainBound::TimestampMs(lo)) => { + ((*v as i128) - (*lo as i128)).max(0) as u128 + } + (DomainBound::Float64(v), DomainBound::Float64(lo)) if v.is_finite() => { + (v - lo).max(0.0) as u128 + } + _ => return None, + }; + Some(u64::try_from(offset / extent).unwrap_or(u64::MAX).min(last)) + }; + match range.map(|r| (index(&r.lo), index(&r.hi))) { + Some((Some(lo), Some(hi))) => (lo.min(hi), lo.max(hi)), + _ => (0, last), } } @@ -265,6 +266,68 @@ mod tests { (ctx, handle) } + /// Every cell a slice can match, or a later write can land in its box, + /// routes to a shard the slice fans out to. + #[test] + fn the_fan_out_covers_the_shard_of_every_cell_in_the_box() { + use nodedb_array::types::coord::value::CoordValue; + let schema = ArraySchemaBuilder::new("grid") + .dim(DimSpec::new( + "chr".to_string(), + DimType::Int64, + Domain::new(DomainBound::Int64(0), DomainBound::Int64(9)), + )) + .dim(DimSpec::new( + "pos".to_string(), + DimType::Int64, + Domain::new(DomainBound::Int64(0), DomainBound::Int64(99)), + )) + .attr(AttrSpec::new("qual".to_string(), AttrType::Float64, true)) + .tile_extents(vec![1, 100]) + .build() + .expect("schema build"); + let box_ranges = vec![ + Some(DimRange::new(DomainBound::Int64(0), DomainBound::Int64(2))), + Some(DimRange::new(DomainBound::Int64(0), DomainBound::Int64(99))), + ]; + let ranges = compute_slice_hilbert_ranges(&schema, &box_ranges, 8); + assert!(!ranges.is_empty(), "a three-tile slice fans out by tile"); + let fanned = nodedb_cluster::distributed_array::array_vshards_for_slice(&ranges, 8, 1024) + .expect("fan-out"); + let mut owners = std::collections::BTreeSet::new(); + for chr in 0..=2 { + for pos in [0, 10, 20, 50, 99] { + let coord = [CoordValue::Int64(chr), CoordValue::Int64(pos)]; + let key = nodedb_array::tile::cell_tile_prefix(&schema, &coord).expect("route"); + let owner = nodedb_cluster::distributed_array::array_vshard_for_tile(key, 8) + .expect("owner"); + assert!( + fanned.contains(&owner), + "cell ({chr}, {pos}) routes to vShard {owner}, outside the fan-out {fanned:?}" + ); + owners.insert(owner); + } + } + assert!(owners.len() > 1, "the slice's tiles spread over shards"); + } + + #[test] + fn a_slice_of_more_tiles_than_the_cap_fans_out_to_every_shard() { + let (_, schema) = make_schema(); + let wide = ArraySchemaBuilder::new("wide") + .dim(DimSpec::new( + "x".to_string(), + DimType::Int64, + Domain::new(DomainBound::Int64(0), DomainBound::Int64(1_000_000)), + )) + .attr(AttrSpec::new("val".to_string(), AttrType::Float64, true)) + .tile_extents(vec![1]) + .build() + .expect("schema build"); + assert!(compute_slice_hilbert_ranges(&wide, &[None], 8).is_empty()); + assert!(!compute_slice_hilbert_ranges(&schema, &[None], 8).is_empty()); + } + #[test] fn single_node_emits_local_array() { let (ctx, _handle) = make_ctx(false); diff --git a/nodedb/src/control/request_tracker.rs b/nodedb/src/control/request_tracker.rs index cc7407edb..e854257df 100644 --- a/nodedb/src/control/request_tracker.rs +++ b/nodedb/src/control/request_tracker.rs @@ -221,7 +221,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -236,7 +236,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/broadcast.rs b/nodedb/src/control/server/broadcast.rs index 29e49c0d8..36684de33 100644 --- a/nodedb/src/control/server/broadcast.rs +++ b/nodedb/src/control/server/broadcast.rs @@ -121,7 +121,7 @@ pub async fn broadcast_to_all_cores_txn( watermark_lsn: outcome.watermark_lsn, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: outcome.read_versions, write_set: Vec::new(), }) } @@ -192,6 +192,7 @@ pub(crate) async fn broadcast_count_to_all_cores_until( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission, }; @@ -273,7 +274,7 @@ pub(crate) async fn broadcast_count_to_all_cores_until( watermark_lsn: max_lsn, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }) } @@ -323,6 +324,7 @@ pub async fn broadcast_register_to_all_cores( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission, }; diff --git a/nodedb/src/control/server/dispatch_utils/dispatch.rs b/nodedb/src/control/server/dispatch_utils/dispatch.rs index 733b96742..b1faa3f65 100644 --- a/nodedb/src/control/server/dispatch_utils/dispatch.rs +++ b/nodedb/src/control/server/dispatch_utils/dispatch.rs @@ -573,7 +573,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) @@ -752,7 +752,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: code.clone().map(Box::new), stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) diff --git a/nodedb/src/control/server/dispatch_utils/durable_write.rs b/nodedb/src/control/server/dispatch_utils/durable_write.rs index 5229ffbd0..3458f1439 100644 --- a/nodedb/src/control/server/dispatch_utils/durable_write.rs +++ b/nodedb/src/control/server/dispatch_utils/durable_write.rs @@ -216,24 +216,24 @@ async fn propose_if_replicable( }; let entry = entry.with_event_source(event_source); let deadline = crate::control::wal_replication::statement_propose_deadline(shared); - let (payload, write_version) = + let (payload, write_versions) = propose_replicated_entry(shared, proposer, entry, deadline).await?; Ok(Some(replicated_write_response( shared, payload, - write_version, + write_versions, ))) } /// The response a committed and applied replicated write answers with. /// -/// `write_version` is the written collection's `coll_write_lsn` after the -/// write. It is the watermark and the read version, so a session can floor a -/// later read at it. +/// `write_versions` are the versions the write stamped. A session floors its +/// later reads of the written vShards at them. A write carries no read +/// watermark. fn replicated_write_response( shared: &SharedState, payload: Vec, - write_version: Lsn, + write_versions: crate::types::ReadVersions, ) -> Response { Response { request_id: RequestId::new(shared.request_id_counter.fetch_add(1, Ordering::Relaxed)), @@ -241,10 +241,10 @@ fn replicated_write_response( attempt: 1, partial: false, payload: payload.into(), - watermark_lsn: write_version, + watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: write_version, + read_versions: write_versions, write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/dispatch_utils/minted/owned.rs b/nodedb/src/control/server/dispatch_utils/minted/owned.rs index 5be212109..daefee3f3 100644 --- a/nodedb/src/control/server/dispatch_utils/minted/owned.rs +++ b/nodedb/src/control/server/dispatch_utils/minted/owned.rs @@ -49,9 +49,10 @@ pub(crate) enum OwnedResponse { /// A response arrived by the deadline. `closed` is the result of closing /// the records from it: a failed cancel returns its error and holds the /// window. A partial response reports `Ok` here, and the task closes the - /// records from the final one. + /// records from the final one. The response is boxed: it is far larger + /// than every other variant. Answered { - response: Response, + response: Box, closed: crate::Result<()>, }, /// The deadline passed first. The task closes the records once the @@ -127,7 +128,7 @@ async fn wait_and_close( match collected { Ok(Ok(response)) if response.partial => { let _ = report.send(OwnedResponse::Answered { - response, + response: Box::new(response), closed: Ok(()), }); forward(resolve_at_final(&wal, owner, final_refusal_key, rx, minted).await); @@ -135,7 +136,10 @@ async fn wait_and_close( Ok(Ok(response)) => { let closed = resolve_on_response(&wal, owner, final_refusal_key, &response, minted).await; - let _ = report.send(OwnedResponse::Answered { response, closed }); + let _ = report.send(OwnedResponse::Answered { + response: Box::new(response), + closed, + }); } Ok(Err(DispatchCollectError::OverBudget { bytes })) => { let _ = report.send(OwnedResponse::OverBudget { bytes }); @@ -199,7 +203,7 @@ mod tests { detail: "duplicate key".into(), })), stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/dispatch_utils/minted/resolve.rs b/nodedb/src/control/server/dispatch_utils/minted/resolve.rs index 53c6baedf..88f0f127e 100644 --- a/nodedb/src/control/server/dispatch_utils/minted/resolve.rs +++ b/nodedb/src/control/server/dispatch_utils/minted/resolve.rs @@ -122,7 +122,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/dispatch_utils/owner_read.rs b/nodedb/src/control/server/dispatch_utils/owner_read.rs index d53847e7f..5050277c6 100644 --- a/nodedb/src/control/server/dispatch_utils/owner_read.rs +++ b/nodedb/src/control/server/dispatch_utils/owner_read.rs @@ -30,7 +30,7 @@ use crate::control::gateway::live_leaders::resolve_live_decision; use crate::control::security::identity::{Permission, required_permission}; use crate::control::server::payload_merge::merge_msgpack_arrays; use crate::control::state::SharedState; -use crate::types::{DatabaseId, Lsn, RequestId, TenantId, TraceId, TxnId, VShardId}; +use crate::types::{DatabaseId, Lsn, ReadVersions, RequestId, TenantId, TraceId, TxnId, VShardId}; /// Where one read runs. pub(crate) struct OwnedReadScope { @@ -95,11 +95,6 @@ async fn route_cluster_read( if scope.linearizable { confirm_linearizable_read(shared, &hosted, deadline).await?; } - // The read's versions are this node's WAL positions. - crate::control::server::shared::session::served_reads::note( - scope.vshard_id.as_u32(), - shared.node_id, - ); Ok(OwnedRead::Local(Box::new(plan))) } ReadPlacement::Owner => { @@ -111,7 +106,7 @@ async fn route_cluster_read( txn_id: scope.txn_id, linearizable: scope.linearizable, }; - owner_response(gateway.execute_internal_with_watermarks(&ctx, plan).await) + owner_response(gateway.execute_internal_outcome(&ctx, plan).await) .map(OwnedRead::Served) } } @@ -151,40 +146,42 @@ pub(crate) fn read_placement( } /// The owner's answer to a read, in the shape a local dispatch returns. +/// +/// A read that found no row answers `NotFound` with the versions it +/// observed, the same as a local miss, so the read-set records the vShard's +/// real version. pub(crate) fn owner_response( - outcome: crate::Result, + outcome: crate::Result, ) -> crate::Result { - match outcome { - Ok((payloads, watermarks, read_version_lsn)) => { - let watermark_lsn = watermarks - .iter() - .map(|(_, lsn)| *lsn) - .max() - .unwrap_or(Lsn::ZERO); - let payload = match payloads.len() { - 0 => Vec::new(), - 1 => payloads.into_iter().next().unwrap_or_default(), - _ => merge_msgpack_arrays(&payloads), - }; - Ok(response( - Status::Ok, - payload, - watermark_lsn, - read_version_lsn, - None, - )) - } - // The local path answers a missing row with a `NotFound` status, not an - // error. The owner's answer keeps that shape. - Err(crate::Error::DataPlane(ErrorCode::NotFound)) => Ok(response( + let outcome = outcome?; + let watermark_lsn = outcome + .shard_watermarks + .iter() + .map(|(_, lsn)| *lsn) + .max() + .unwrap_or(Lsn::ZERO); + if outcome.not_found { + return Ok(response( Status::Error, Vec::new(), - Lsn::ZERO, - Lsn::ZERO, + watermark_lsn, + outcome.read_versions, Some(ErrorCode::NotFound), - )), - Err(error) => Err(error), + )); } + let payloads = outcome.payloads; + let payload = match payloads.len() { + 0 => Vec::new(), + 1 => payloads.into_iter().next().unwrap_or_default(), + _ => merge_msgpack_arrays(&payloads), + }; + Ok(response( + Status::Ok, + payload, + watermark_lsn, + outcome.read_versions, + None, + )) } /// Prepare a read-only pass that must run on this node's cores. @@ -233,7 +230,13 @@ fn leads_vshard(shared: &SharedState, vshard_id: VShardId) -> bool { /// A successful response carrying `payload`, in the shape a local dispatch /// returns. pub(crate) fn ok_payload_response(payload: Payload) -> Response { - response(Status::Ok, payload.to_vec(), Lsn::ZERO, Lsn::ZERO, None) + response( + Status::Ok, + payload.to_vec(), + Lsn::ZERO, + ReadVersions::new(), + None, + ) } /// A `NotFound` refusal, the shape a local dispatch returns for a read that @@ -243,7 +246,7 @@ pub(crate) fn not_found_response() -> Response { Status::Error, Vec::new(), Lsn::ZERO, - Lsn::ZERO, + ReadVersions::new(), Some(ErrorCode::NotFound), ) } @@ -252,7 +255,7 @@ fn response( status: Status, payload: Vec, watermark_lsn: Lsn, - read_version_lsn: Lsn, + read_versions: ReadVersions, error_code: Option, ) -> Response { Response { @@ -264,7 +267,48 @@ fn response( watermark_lsn, error_code: error_code.map(Box::new), stage_vote: None, - read_version_lsn, + read_versions, write_set: Vec::new(), } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::gateway::outcome::GatewayOutcome; + + fn outcome(not_found: bool) -> GatewayOutcome { + GatewayOutcome { + payloads: Vec::new(), + shard_watermarks: vec![(VShardId::new(3), Lsn::new(7))], + read_versions: ReadVersions::single( + VShardId::new(3), + nodedb_types::WriteVersion::logged(1, 40), + ), + not_found, + } + } + + /// A remote read that found no row answers like a local miss, with the + /// version it observed for the read-set. + #[test] + fn an_owner_miss_keeps_its_verdict_and_its_version() { + let response = owner_response(Ok(outcome(true))).expect("an owner miss answers"); + assert_eq!(response.status, Status::Error); + assert_eq!(response.error_code.as_deref(), Some(&ErrorCode::NotFound)); + assert_eq!( + response.read_versions.of(VShardId::new(3)), + Some(nodedb_types::WriteVersion::logged(1, 40)) + ); + } + + #[test] + fn an_owner_answer_keeps_its_version() { + let response = owner_response(Ok(outcome(false))).expect("an owner answer"); + assert_eq!(response.status, Status::Ok); + assert_eq!( + response.read_versions.of(VShardId::new(3)), + Some(nodedb_types::WriteVersion::logged(1, 40)) + ); + } +} diff --git a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/answer.rs b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/answer.rs index 01468577c..b5fbd302e 100644 --- a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/answer.rs +++ b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/answer.rs @@ -39,7 +39,7 @@ impl Answer { .await; Ok(match collected { Ok(Ok(response)) => OwnedResponse::Answered { - response, + response: Box::new(response), closed: Ok(()), }, Ok(Err(DispatchCollectError::OverBudget { bytes })) => { diff --git a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/dispatch.rs b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/dispatch.rs index f289df5a6..c7fb27a57 100644 --- a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/dispatch.rs +++ b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/dispatch.rs @@ -43,6 +43,9 @@ pub(super) struct DispatchTarget { pub resolved_now_ms: Option, /// The write's commit HLC, which dates every event it emits. pub commit_hlc: u64, + /// The log position of the data-group entry the write applies, its + /// version on every replica. `None` for a write that applies no entry. + pub entry_version: Option, pub admission: Admission, /// The record group the write journals its write set into. pub journal: Option, @@ -99,6 +102,7 @@ pub(super) async fn dispatch_to_data_plane( wal_lsn: target.wal_lsn, resolved_now_ms: target.resolved_now_ms, commit_hlc: Some(target.commit_hlc), + entry_version: target.entry_version, admission: target.admission, }; diff --git a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/driver.rs b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/driver.rs index 7829bd0d3..65cb2f4a1 100644 --- a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/driver.rs +++ b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/driver.rs @@ -393,6 +393,11 @@ pub(crate) async fn enqueue_write( wal_lsn, resolved_now_ms, commit_hlc, + // A write that applies a data-group entry versions its rows by + // the entry's position, which every replica shares. + entry_version: wal_append_outcome.change_position.map(|position| { + nodedb_types::WriteVersion::logged(position.epoch, position.log_index) + }), admission, journal, }, diff --git a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/response.rs b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/response.rs index d4ce8d8df..a4e3df114 100644 --- a/nodedb/src/control/server/dispatch_utils/submit_write/funnel/response.rs +++ b/nodedb/src/control/server/dispatch_utils/submit_write/funnel/response.rs @@ -122,7 +122,7 @@ pub(super) async fn collect_classify_and_finish( OwnedResponse::Answered { response, closed } => { // A failed cancel holds the window and fails the write here. closed?; - response + *response } OwnedResponse::DeadlineExceeded => { observe(shared); diff --git a/nodedb/src/control/server/exchange/all_cores/base_snapshot.rs b/nodedb/src/control/server/exchange/all_cores/base_snapshot.rs index 0fd10ea46..363eaa917 100644 --- a/nodedb/src/control/server/exchange/all_cores/base_snapshot.rs +++ b/nodedb/src/control/server/exchange/all_cores/base_snapshot.rs @@ -95,7 +95,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/exchange/all_cores/bsp.rs b/nodedb/src/control/server/exchange/all_cores/bsp.rs index 7ea8ab90b..ebc1d96a2 100644 --- a/nodedb/src/control/server/exchange/all_cores/bsp.rs +++ b/nodedb/src/control/server/exchange/all_cores/bsp.rs @@ -60,7 +60,8 @@ pub(super) async fn fan_bsp_all_cores( Ok(NodeLevelResult { payload, watermark_lsn, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), + not_found: false, }) } diff --git a/nodedb/src/control/server/exchange/all_cores/dispatch.rs b/nodedb/src/control/server/exchange/all_cores/dispatch.rs index 0924e0c0e..10bb2d30c 100644 --- a/nodedb/src/control/server/exchange/all_cores/dispatch.rs +++ b/nodedb/src/control/server/exchange/all_cores/dispatch.rs @@ -18,10 +18,13 @@ use super::wcc::fan_wcc_all_cores; pub struct NodeLevelResult { pub payload: Vec, pub watermark_lsn: Lsn, - /// Max per-collection read-version LSN of the fanned read. `Lsn::ZERO` for - /// non-read results. Distinct from `watermark_lsn` — the comparand for + /// The read versions the fanned read observed, one per vShard. Empty for + /// non-read results. Distinct from `watermark_lsn`: the comparand for /// cross-shard OCC read validation. - pub read_version_lsn: Lsn, + pub read_versions: crate::types::ReadVersions, + /// The plan's one owning core refused it with `NotFound`: a read that + /// found no row. `read_versions` holds the versions it observed. + pub not_found: bool, } /// Fan `plan` across all local Data-Plane cores, merge per-core payloads, and @@ -77,7 +80,8 @@ pub(crate) async fn execute_plan_all_local_cores( Ok(NodeLevelResult { payload: envelope, watermark_lsn: outcome.watermark_lsn, - read_version_lsn: Lsn::ZERO, + read_versions: outcome.read_versions, + not_found: false, }) } @@ -261,16 +265,18 @@ async fn generic_gather( ) .await?; return Ok(NodeLevelResult { + not_found: crate::control::local_dispatch::is_not_found(&resp), payload: resp.payload.to_vec(), watermark_lsn: resp.watermark_lsn, - read_version_lsn: resp.read_version_lsn, + read_versions: resp.read_versions, }); } let outcome = gather_all_cores(state, tenant_id, database_id, plan, trace_id, txn_id).await?; Ok(NodeLevelResult { payload: outcome.merged_array, watermark_lsn: outcome.watermark_lsn, - read_version_lsn: outcome.read_version_lsn, + read_versions: outcome.read_versions, + not_found: false, }) } @@ -300,22 +306,21 @@ async fn single_blob_gather( gather_every_core(state, tenant_id, database_id, plan, trace_id, "single-blob").await?; let mut watermark_lsn = Lsn::ZERO; - let mut read_version_lsn = Lsn::ZERO; + let mut read_versions = crate::types::ReadVersions::new(); let mut payloads: Vec> = Vec::with_capacity(responses.len()); for resp in responses { if resp.watermark_lsn > watermark_lsn { watermark_lsn = resp.watermark_lsn; } - if resp.read_version_lsn > read_version_lsn { - read_version_lsn = resp.read_version_lsn; - } + read_versions.merge(&resp.read_versions); payloads.push(resp.payload.as_ref().to_vec()); } Ok(NodeLevelResult { payload: pick_single_blob(payloads, op)?, watermark_lsn, - read_version_lsn, + read_versions, + not_found: false, }) } diff --git a/nodedb/src/control/server/exchange/all_cores/home_versions.rs b/nodedb/src/control/server/exchange/all_cores/home_versions.rs index 2b46f11f4..950b81069 100644 --- a/nodedb/src/control/server/exchange/all_cores/home_versions.rs +++ b/nodedb/src/control/server/exchange/all_cores/home_versions.rs @@ -4,9 +4,8 @@ //! vShards it owns, under this node's leader lease on each probe's group. //! //! A write to a vShard runs on the one core the dispatcher's router assigns -//! it, and raises that core's watermark and the collection's write floor -//! there. So a probe is answered by that core alone. A write that another core -//! took, to another vShard, never moves the answer. +//! it, and raises the vShard's versions there. So a probe is answered by that +//! core alone. A write to another vShard never moves the answer. //! //! A node that lost leadership of a group can miss writes a newer leader //! committed. So a probe is answered only while this node holds its group's @@ -104,7 +103,8 @@ pub(super) async fn fan_home_versions( Ok(NodeLevelResult { payload, watermark_lsn, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), + not_found: false, }) } diff --git a/nodedb/src/control/server/exchange/all_cores/snapshot.rs b/nodedb/src/control/server/exchange/all_cores/snapshot.rs index 90ecab08f..5eef3a328 100644 --- a/nodedb/src/control/server/exchange/all_cores/snapshot.rs +++ b/nodedb/src/control/server/exchange/all_cores/snapshot.rs @@ -193,7 +193,8 @@ pub(super) fn merge_core_snapshots( Ok(NodeLevelResult { payload, watermark_lsn, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), + not_found: false, }) } diff --git a/nodedb/src/control/server/exchange/all_cores/wcc.rs b/nodedb/src/control/server/exchange/all_cores/wcc.rs index 13f5285d6..9a0a65d75 100644 --- a/nodedb/src/control/server/exchange/all_cores/wcc.rs +++ b/nodedb/src/control/server/exchange/all_cores/wcc.rs @@ -59,7 +59,8 @@ pub(super) async fn fan_wcc_all_cores( Ok(NodeLevelResult { payload, watermark_lsn, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), + not_found: false, }) } diff --git a/nodedb/src/control/server/exchange/cluster_leaf.rs b/nodedb/src/control/server/exchange/cluster_leaf.rs index 5b67ff500..7b53b7b9f 100644 --- a/nodedb/src/control/server/exchange/cluster_leaf.rs +++ b/nodedb/src/control/server/exchange/cluster_leaf.rs @@ -185,7 +185,7 @@ fn outcome_of(payload: Vec) -> GatherOutcome { raw: payload, merged_array, watermark_lsn: Lsn::ZERO, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), shard_watermarks: Vec::new(), } } diff --git a/nodedb/src/control/server/exchange/core_outcome.rs b/nodedb/src/control/server/exchange/core_outcome.rs index 5fed5382c..a1d3c66c7 100644 --- a/nodedb/src/control/server/exchange/core_outcome.rs +++ b/nodedb/src/control/server/exchange/core_outcome.rs @@ -28,6 +28,19 @@ pub(crate) fn classify_core_response(collected: crate::Result) -> Core Ok(Some(resp)) } +/// The versions every core reported, a `NotFound` refusal's included. A +/// refusal observed its vShards with no matching row, and a read validates +/// that observation at the versions the refusal reported. +pub(crate) fn reported_versions<'a>( + collected: impl IntoIterator>, +) -> crate::types::ReadVersions { + let mut versions = crate::types::ReadVersions::new(); + for resp in collected.into_iter().flatten() { + versions.merge(&resp.read_versions); + } + versions +} + /// Require every core to answer, and return the answers in core order. /// /// The first error in core order fails the whole gather. It crosses as the @@ -60,7 +73,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/exchange/gather.rs b/nodedb/src/control/server/exchange/gather.rs index 7a1ada834..3cf519820 100644 --- a/nodedb/src/control/server/exchange/gather.rs +++ b/nodedb/src/control/server/exchange/gather.rs @@ -20,7 +20,9 @@ use futures::future::join_all; use crate::bridge::envelope::{PhysicalPlan, Priority, Request, Response}; use crate::control::arrow_convert; use crate::control::gateway::core::QueryContext; -use crate::control::server::exchange::core_outcome::{classify_core_response, require_every_core}; +use crate::control::server::exchange::core_outcome::{ + classify_core_response, reported_versions, require_every_core, +}; use crate::control::server::exchange::read_scope::ReadScope; use crate::control::server::payload_merge::{encode_msgpack_array, extract_msgpack_elements}; use crate::control::server::result_stream::ResultStream; @@ -126,6 +128,7 @@ pub(crate) fn eager_dispatch_to_all_cores_until( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), @@ -154,12 +157,11 @@ pub struct GatherOutcome { /// Maximum watermark LSN seen across all responding cores. Retained as the /// scalar fence value for Strong-consistency callers that need one LSN. pub watermark_lsn: Lsn, - /// Max per-collection read-version LSN across all responding cores. A read - /// homes to exactly ONE core, so the non-owning cores report `Lsn::ZERO` and - /// the owning core's value dominates the fold — yielding the scanned - /// collection's `coll_write_lsn` at read time (the sound comparand for - /// cross-shard OCC read validation, distinct from `watermark_lsn`). - pub read_version_lsn: Lsn, + /// The read versions every responding core reported, one per vShard. A + /// vShard lives on ONE core, so only that core reports it: the scanned + /// collection's version at read time (the comparand for cross-shard OCC + /// read validation, distinct from `watermark_lsn`). + pub read_versions: crate::types::ReadVersions, /// Per-shard watermark LSNs — one `(vshard, watermark_lsn)` per responding /// core, NOT collapsed to the max. The transaction read-set records one /// entry per participating shard from this, so a predicate read fanned over @@ -219,6 +221,9 @@ pub(crate) async fn gather_all_cores( }); let results: Vec<(usize, crate::Result)> = join_all(response_futures).await; + // The read versions of every core, one per vShard: a vShard lives on ONE + // core, so only that core reports it. + let read_versions = reported_versions(results.iter().map(|(_, result)| result)); let answered = require_every_core(results.into_iter().map(|(core_id, result)| { classify_core_response(result).map(|resp| resp.map(|resp| (core_id, resp))) }))?; @@ -226,11 +231,6 @@ pub(crate) async fn gather_all_cores( let mut raw = Vec::new(); let mut all_elements: Vec> = Vec::new(); let mut max_lsn = Lsn::ZERO; - // Max-fold of the per-collection read-version across cores: a collection - // homes to ONE core, so non-owning cores contribute `Lsn::ZERO` and the - // owning core's value dominates. Kept as a single scalar (not per-core) — - // the read plan targets one collection, so one non-zero value survives. - let mut max_read_version = Lsn::ZERO; let mut shard_watermarks: Vec<(VShardId, Lsn)> = Vec::new(); for (core_id, resp) in answered { @@ -242,9 +242,6 @@ pub(crate) async fn gather_all_cores( if resp.watermark_lsn > max_lsn { max_lsn = resp.watermark_lsn; } - if resp.read_version_lsn > max_read_version { - max_read_version = resp.read_version_lsn; - } if resp.payload.is_empty() { continue; @@ -261,7 +258,7 @@ pub(crate) async fn gather_all_cores( raw, merged_array, watermark_lsn: max_lsn, - read_version_lsn: max_read_version, + read_versions, shard_watermarks, }) } @@ -382,8 +379,10 @@ pub(crate) async fn gather_all_vshards( // The gateway already fails with a typed `crate::Error` — a shard's // `Error::DataPlane` code included. Re-wrapping it in `Dispatch` will // rewrite every such verdict as SQLSTATE XX000, so it passes through. - let (payloads, shard_watermarks, read_version_lsn): (Vec>, Vec<(VShardId, Lsn)>, Lsn) = - Box::pin(gateway.execute_internal_with_watermarks(&ctx, plan)).await?; + let (payloads, shard_watermarks, read_versions) = + Box::pin(gateway.execute_internal_outcome(&ctx, plan)) + .await? + .into_parts(); let mut all_elements: Vec> = Vec::new(); let mut raw = Vec::new(); @@ -407,7 +406,7 @@ pub(crate) async fn gather_all_vshards( raw, merged_array, watermark_lsn, - read_version_lsn, + read_versions, shard_watermarks, }) } diff --git a/nodedb/src/control/server/exchange/owning_core.rs b/nodedb/src/control/server/exchange/owning_core.rs index a90eaea45..3701141e9 100644 --- a/nodedb/src/control/server/exchange/owning_core.rs +++ b/nodedb/src/control/server/exchange/owning_core.rs @@ -27,7 +27,7 @@ use super::gather::GatherOutcome; /// collection's rows. /// /// The returned outcome carries that core's own `watermark_lsn` / -/// `read_version_lsn` and exactly one `shard_watermarks` entry keyed to the +/// `read_versions` and exactly one `shard_watermarks` entry keyed to the /// collection's vShard — matching the cluster `dispatch_local` path so an /// in-transaction read records the same OCC read-set entry the write-set uses /// (writes home to the same `CollectionKey` vShard). Aggregate @@ -60,7 +60,7 @@ pub async fn gather_single_owning_core( raw: payload_bytes.to_vec(), merged_array, watermark_lsn: resp.watermark_lsn, - read_version_lsn: resp.read_version_lsn, + read_versions: resp.read_versions.clone(), shard_watermarks: vec![(vshard_id, resp.watermark_lsn)], }) } diff --git a/nodedb/src/control/server/exchange/received.rs b/nodedb/src/control/server/exchange/received.rs index 322071d70..7fe890862 100644 --- a/nodedb/src/control/server/exchange/received.rs +++ b/nodedb/src/control/server/exchange/received.rs @@ -72,9 +72,10 @@ pub(crate) async fn execute_received_plan( ) .await?; Ok(NodeLevelResult { + not_found: crate::control::local_dispatch::is_not_found(&resp), payload: resp.payload.to_vec(), watermark_lsn: resp.watermark_lsn, - read_version_lsn: resp.read_version_lsn, + read_versions: resp.read_versions, }) } ReceivedRoute::AllCores => { diff --git a/nodedb/src/control/server/exchange/resolve/capture.rs b/nodedb/src/control/server/exchange/resolve/capture.rs index fa49e558f..a4c5fc886 100644 --- a/nodedb/src/control/server/exchange/resolve/capture.rs +++ b/nodedb/src/control/server/exchange/resolve/capture.rs @@ -4,7 +4,7 @@ use nodedb_physical::physical_plan::PhysicalPlan; -use crate::types::Lsn; +use crate::types::ReadVersions; /// One base collection's read observation from a distributed read whose result /// is materialized on the coordinator. @@ -13,16 +13,16 @@ use crate::types::Lsn; /// collection under a root `Exchange{Gather}`, including both sides of a /// gathered `HashJoin`) and the SHUFFLE path (`resolve::shuffle` / /// `resolve::shuffle_aggregate` — one per join side). Each carries that -/// collection's own bare full-collection scan plan and the REAL per-collection -/// read-version LSN the gather/producers observed. The record seam re-derives -/// the collection / engine / read key from `scan_plan` (a single-collection -/// scan, so it is NOT collapsed to just the left side the way a `HashJoin` plan -/// is) and stamps the entry at `read_version_lsn`, so the commit-time OCC +/// collection's own bare full-collection scan plan and the REAL read versions +/// the gather/producers observed. The record seam re-derives the collection / +/// engine / read key from `scan_plan` (a single-collection scan, so it is NOT +/// collapsed to just the left side the way a `HashJoin` plan is) and stamps +/// the entry at the collection's version in `read_versions`, so the commit-time OCC /// validator re-homes and revalidates each collection's vshard independently. /// This closes the serializability hole where a build-side (or non-left) /// collection never appeared in the read-set and a concurrent write to it went /// undetected. pub struct DistributedReadCapture { pub scan_plan: PhysicalPlan, - pub read_version_lsn: Lsn, + pub read_versions: ReadVersions, } diff --git a/nodedb/src/control/server/exchange/resolve/exchange/gather_arm.rs b/nodedb/src/control/server/exchange/resolve/exchange/gather_arm.rs index cd31c918b..315c71364 100644 --- a/nodedb/src/control/server/exchange/resolve/exchange/gather_arm.rs +++ b/nodedb/src/control/server/exchange/resolve/exchange/gather_arm.rs @@ -97,8 +97,8 @@ pub(super) async fn resolve_gather( let outcome: GatherOutcome = gather_all_vshards(state, child, ctx).await?; - // Record the probe/single-collection read at its OWN observed - // read-version (the gathered collection's `coll_write_lsn`), scoped to + // Record the probe/single-collection read at its OWN observed read + // versions (the gathered collection's version per vShard), scoped to // a bare single-collection scan so the commit-time OCC validator // re-homes and revalidates exactly that collection's vshard. A // `HashJoin` plan will otherwise collapse to the left collection @@ -114,7 +114,7 @@ pub(super) async fn resolve_gather( { captures.push(DistributedReadCapture { scan_plan, - read_version_lsn: outcome.read_version_lsn, + read_versions: outcome.read_versions.clone(), }); } @@ -124,7 +124,7 @@ pub(super) async fn resolve_gather( outcome.merged_array }; Ok(Resolved::Gathered( - outcome_to_response(payload, outcome.watermark_lsn, outcome.read_version_lsn), + outcome_to_response(payload, outcome.watermark_lsn, outcome.read_versions), outcome.shard_watermarks, std::mem::take(captures), )) @@ -143,7 +143,7 @@ pub(super) async fn resolve_broadcast( outcome_to_response( outcome.merged_array, outcome.watermark_lsn, - outcome.read_version_lsn, + outcome.read_versions, ), outcome.shard_watermarks, std::mem::take(captures), diff --git a/nodedb/src/control/server/exchange/resolve/exchange/post_process_arm.rs b/nodedb/src/control/server/exchange/resolve/exchange/post_process_arm.rs index 4a6027f2c..35ae1c939 100644 --- a/nodedb/src/control/server/exchange/resolve/exchange/post_process_arm.rs +++ b/nodedb/src/control/server/exchange/resolve/exchange/post_process_arm.rs @@ -225,7 +225,7 @@ pub(super) async fn materialize_child_rows( { captures.push(DistributedReadCapture { scan_plan, - read_version_lsn: outcome.read_version_lsn, + read_versions: outcome.read_versions.clone(), }); } diff --git a/nodedb/src/control/server/exchange/resolve/join_input.rs b/nodedb/src/control/server/exchange/resolve/join_input.rs index 515dea2f4..9f1130e6e 100644 --- a/nodedb/src/control/server/exchange/resolve/join_input.rs +++ b/nodedb/src/control/server/exchange/resolve/join_input.rs @@ -117,7 +117,7 @@ pub(super) async fn resolve_join_input( { captures.push(DistributedReadCapture { scan_plan, - read_version_lsn: outcome.read_version_lsn, + read_versions: outcome.read_versions.clone(), }); } let merged = if as_aggregate { @@ -209,7 +209,7 @@ pub(super) async fn gather_join_build_side( if let Some(scan_plan) = capture_plan { captures.push(DistributedReadCapture { scan_plan, - read_version_lsn: outcome.read_version_lsn, + read_versions: outcome.read_versions.clone(), }); } diff --git a/nodedb/src/control/server/exchange/resolve/peers.rs b/nodedb/src/control/server/exchange/resolve/peers.rs index 36a5f90c7..52f7333c2 100644 --- a/nodedb/src/control/server/exchange/resolve/peers.rs +++ b/nodedb/src/control/server/exchange/resolve/peers.rs @@ -93,23 +93,23 @@ pub(super) fn distinct_data_node_count(routing: &RoutingTable) -> usize { } /// Send one `ShuffleProduceRequest` and map the reply / RPC error to a typed -/// coordinator error, returning the producer's observed per-collection -/// read-version LSN on a clean produce. Fail-fast: a producer-reported terminal -/// error aborts. Shared by both the shuffle-JOIN and shuffle-AGGREGATE -/// resolvers, which each max-fold the returned LSN over their producers. +/// coordinator error, returning the producer's observed read versions on a +/// clean produce. Fail-fast: a producer-reported terminal error aborts. Shared +/// by both the shuffle-JOIN and shuffle-AGGREGATE resolvers, which each fold +/// the returned versions over their producers. pub(super) async fn send_produce( transport: &nodedb_cluster::NexarTransport, node: u64, req: ShuffleProduceRequest, -) -> crate::Result { +) -> crate::Result { match transport .send_rpc(node, RaftRpc::ShuffleProduceRequest(req)) .await { Ok(RaftRpc::ShuffleProduceResponse(ShuffleProduceResponse { error: None, - read_version_lsn, - })) => Ok(read_version_lsn), + read_versions, + })) => Ok(crate::types::ReadVersions::from_wire(&read_versions)), Ok(RaftRpc::ShuffleProduceResponse(ShuffleProduceResponse { error: Some(e), .. })) => { Err(crate::Error::Internal { detail: format!("shuffle produce failed on node {node}: {e:?}"), diff --git a/nodedb/src/control/server/exchange/resolve/shuffle.rs b/nodedb/src/control/server/exchange/resolve/shuffle.rs index f44818d68..af0f31fb1 100644 --- a/nodedb/src/control/server/exchange/resolve/shuffle.rs +++ b/nodedb/src/control/server/exchange/resolve/shuffle.rs @@ -294,23 +294,23 @@ pub async fn resolve_shuffle_join( probe_produce_futures.push(send_produce(transport, node, req)); } // Await ALL producers (both sides CONCURRENTLY); any error fails the whole - // shuffle (no partial join). Max-fold each side's producers' observed - // per-collection read-version LSN independently: build ↔ right_collection, - // probe ↔ left_collection. Each side's producers scan the SAME single - // collection, so its max is that collection's `coll_write_lsn` at read time — - // the sound OCC read-version comparand recorded for that side. + // shuffle (no partial join). Fold each side's producers' observed read + // versions independently: build ↔ right_collection, probe ↔ + // left_collection. Each side's producers scan the SAME single collection, + // so the fold holds that collection's version on each vShard at read time: + // the OCC read-version comparand recorded for that side. let (build_results, probe_results) = join( join_all(build_produce_futures), join_all(probe_produce_futures), ) .await; - let mut build_rv: u64 = 0; + let mut build_rv = crate::types::ReadVersions::new(); for result in build_results { - build_rv = build_rv.max(result?); + build_rv.merge(&result?); } - let mut probe_rv: u64 = 0; + let mut probe_rv = crate::types::ReadVersions::new(); for result in probe_results { - probe_rv = probe_rv.max(result?); + probe_rv.merge(&result?); } // 8. After ALL producers succeed, dispatch consumers CONCURRENTLY — one per @@ -364,7 +364,7 @@ pub async fn resolve_shuffle_join( // Two per-side read captures carry each side's REAL observed read version: // the probe capture (left_collection) at `probe_rv`, the build capture // (right_collection) at `build_rv` — folded from the producers' - // `ShuffleProduceResponse.read_version_lsn`. The record seam records one + // `ShuffleProduceResponse.read_versions`. The record seam records one // read-set entry per capture, re-homing and revalidating each side's vshard // independently, so a concurrent write to EITHER side between the in-txn read // and commit is detected (the build side is recorded too). The response's own scalar stays `ZERO`: the captures @@ -374,15 +374,15 @@ pub async fn resolve_shuffle_join( let captures = vec![ DistributedReadCapture { scan_plan: probe_scan, - read_version_lsn: Lsn::new(probe_rv), + read_versions: probe_rv, }, DistributedReadCapture { scan_plan: build_scan, - read_version_lsn: Lsn::new(build_rv), + read_versions: build_rv, }, ]; Ok(Resolved::Gathered( - outcome_to_response(merged, Lsn::ZERO, Lsn::ZERO), + outcome_to_response(merged, Lsn::ZERO, crate::types::ReadVersions::new()), Vec::new(), captures, )) diff --git a/nodedb/src/control/server/exchange/resolve/shuffle_aggregate.rs b/nodedb/src/control/server/exchange/resolve/shuffle_aggregate.rs index f6e94d43d..1ad4e28d6 100644 --- a/nodedb/src/control/server/exchange/resolve/shuffle_aggregate.rs +++ b/nodedb/src/control/server/exchange/resolve/shuffle_aggregate.rs @@ -257,13 +257,13 @@ pub async fn resolve_shuffle_aggregate( produce_futures.push(send_produce(transport, node, req)); } // Await ALL producers; any error fails the whole shuffle (no partial result). - // Max-fold every producer's observed per-collection read-version LSN: the - // producers all scan the SAME single source collection, so the max is that - // collection's `coll_write_lsn` at read time — the sound comparand the - // coordinator records for cross-shard OCC read validation of this aggregate. - let mut max_read_version_lsn: u64 = 0; + // Fold every producer's observed read versions: the producers all scan the + // SAME single source collection, so the fold holds that collection's + // version on each vShard at read time: the comparand the coordinator + // records for cross-shard OCC read validation of this aggregate. + let mut read_versions = crate::types::ReadVersions::new(); for result in join_all(produce_futures).await { - max_read_version_lsn = max_read_version_lsn.max(result?); + read_versions.merge(&result?); } // 7. After ALL producers succeed, dispatch consumers CONCURRENTLY — one per @@ -332,16 +332,16 @@ pub async fn resolve_shuffle_aggregate( } let merged = encode_msgpack_array(&elements); - // The producers report the source collection's `coll_write_lsn` at read time - // on their `ShuffleProduceResponse`; the max-fold above is the read version - // this aggregate observed. Pass it as the OCC read-version comparand so an + // The producers report the source collection's versions at read time on + // their `ShuffleProduceResponse`; the fold above is what this aggregate + // observed. Pass it as the OCC read-version comparand so an // in-transaction distributed aggregate records a sound read-set entry (the // aggregate is single-collection, so `record_read_set` attributes it to the // right collection). The core-global `watermark_lsn` is NOT threaded through // the shuffle transport and stays `ZERO`; using it as the read version will // skip required aborts (it advances on writes to ANY collection). Ok(Resolved::Gathered( - outcome_to_response(merged, Lsn::ZERO, Lsn::new(max_read_version_lsn)), + outcome_to_response(merged, Lsn::ZERO, read_versions), Vec::new(), Vec::new(), )) diff --git a/nodedb/src/control/server/exchange/response.rs b/nodedb/src/control/server/exchange/response.rs index 541836515..25503976f 100644 --- a/nodedb/src/control/server/exchange/response.rs +++ b/nodedb/src/control/server/exchange/response.rs @@ -4,7 +4,7 @@ use crate::bridge::envelope::{Response, Status}; use crate::control::server::result_stream::ResultStream; -use crate::types::{Lsn, RequestId}; +use crate::types::{Lsn, ReadVersions, RequestId}; /// Materialize a [`ResultStream`] into a synthetic successful Response. /// @@ -14,26 +14,30 @@ use crate::types::{Lsn, RequestId}; pub(crate) async fn stream_to_response(stream: ResultStream) -> crate::Result { let (merged, watermark_lsn) = crate::control::server::result_stream::materialize(stream).await?; - // A streamed result carries no per-collection read-version: the stream's - // frames carry only per-batch watermarks, and nothing consumes a read-version - // from this path — the streaming branch is gated on `txn_id.is_none()` - // (`resolve/exchange.rs`), so a stream never serves an in-transaction read and - // never reaches the read-set capture that would compare one. - Ok(outcome_to_response(merged, watermark_lsn, Lsn::ZERO)) + // A streamed result carries no read versions: the stream's frames carry + // only per-batch watermarks, and nothing consumes a read version from this + // path. The streaming branch is gated on `txn_id.is_none()` + // (`resolve/exchange.rs`), so a stream never serves an in-transaction read + // and never reaches the read-set capture. + Ok(outcome_to_response( + merged, + watermark_lsn, + ReadVersions::new(), + )) } /// Build a synthetic successful Response from a gathered merged-array payload. /// -/// `read_version_lsn` is the gathered collection's `coll_write_lsn` at read time -/// (`GatherOutcome::read_version_lsn`) — the comparand cross-shard OCC read -/// validation consumes. It is distinct from `watermark_lsn` (the core-global -/// fence) and must be passed through: dropping it to `Lsn::ZERO` silently strips -/// the read's version and leaves validation to compare against whatever floor the +/// `read_versions` are the gathered read's versions at read time +/// (`GatherOutcome::read_versions`): the comparand cross-shard OCC read +/// validation consumes. They are distinct from `watermark_lsn` (the core-global +/// fence) and must be passed through: dropping them silently strips the read's +/// versions and leaves validation to compare against whatever floor the /// session's own writes supply. pub(super) fn outcome_to_response( merged_array: Vec, watermark_lsn: Lsn, - read_version_lsn: Lsn, + read_versions: ReadVersions, ) -> Response { Response { request_id: RequestId::new(0), @@ -44,7 +48,7 @@ pub(super) fn outcome_to_response( watermark_lsn, error_code: None, stage_vote: None, - read_version_lsn, + read_versions, write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/graph_dispatch/bfs.rs b/nodedb/src/control/server/graph_dispatch/bfs.rs index 333c962c2..77104e945 100644 --- a/nodedb/src/control/server/graph_dispatch/bfs.rs +++ b/nodedb/src/control/server/graph_dispatch/bfs.rs @@ -90,7 +90,7 @@ pub async fn cross_core_bfs_with_options( &mut presence_reads, ) .await?; - presence_reads.publish(shared, tenant_id, database_id, None); + presence_reads.publish(tenant_id, database_id, None); frontier.retain(|node| present.contains(node) && visited.insert(node.clone())); all_discovered.extend(frontier.iter().cloned()); @@ -130,12 +130,7 @@ pub async fn cross_core_bfs_with_options( } // Every vShard the walk expanded joins the transaction read-set. - reads.publish( - shared, - tenant_id, - database_id, - collection.map(str::to_owned), - ); + reads.publish(tenant_id, database_id, collection.map(str::to_owned)); Ok(ok_response(encode_path(&all_discovered)?)) } diff --git a/nodedb/src/control/server/graph_dispatch/bsp_pagerank/coord.rs b/nodedb/src/control/server/graph_dispatch/bsp_pagerank/coord.rs index 807823fbf..6ad7529de 100644 --- a/nodedb/src/control/server/graph_dispatch/bsp_pagerank/coord.rs +++ b/nodedb/src/control/server/graph_dispatch/bsp_pagerank/coord.rs @@ -166,19 +166,14 @@ pub async fn run_bsp_pagerank( // The count phase reads every owner's partition first. A write after it // on any owned vShard changes what later supersteps read, so the count - // phase's watermarks are the ones the transaction read-set keeps. + // phase's versions are the ones the transaction read-set keeps. let mut reads = ShardReadLog::new(); for count in &counts { if let Some(target) = targets.iter().find(|t| t.node_id == count.node_id) { - reads.note( - target.owned_vshards.iter().copied(), - count.watermark_lsn, - target.node_id, - ); + reads.note(target.owned_vshards.iter().copied(), &count.read_versions); } } reads.publish( - state, tenant_id, database_id, Some(qualified(database_id, ¶ms.collection)), diff --git a/nodedb/src/control/server/graph_dispatch/bsp_pagerank/scatter.rs b/nodedb/src/control/server/graph_dispatch/bsp_pagerank/scatter.rs index 61ff23148..9ecb5432e 100644 --- a/nodedb/src/control/server/graph_dispatch/bsp_pagerank/scatter.rs +++ b/nodedb/src/control/server/graph_dispatch/bsp_pagerank/scatter.rs @@ -59,8 +59,8 @@ pub(super) struct ShardDispatch { pub(super) struct ShardResult { pub(super) node_id: u64, pub(super) result: BspSuperstepResult, - /// The highest watermark the node's cores served the superstep at. - pub(super) watermark_lsn: crate::types::Lsn, + /// The versions the node's cores reported for the superstep. + pub(super) read_versions: crate::types::ReadVersions, } /// Parameters for [`scatter_superstep`]. @@ -139,7 +139,7 @@ pub(super) async fn scatter_superstep( Ok::(ShardResult { node_id, result, - watermark_lsn: read.watermark_lsn, + read_versions: read.read_versions, }) }) }); diff --git a/nodedb/src/control/server/graph_dispatch/bsp_wcc/coord.rs b/nodedb/src/control/server/graph_dispatch/bsp_wcc/coord.rs index 4657099b5..3a1c80607 100644 --- a/nodedb/src/control/server/graph_dispatch/bsp_wcc/coord.rs +++ b/nodedb/src/control/server/graph_dispatch/bsp_wcc/coord.rs @@ -77,19 +77,14 @@ pub async fn run_bsp_wcc( .map(|sr| (sr.node_id, sr.result.system_as_of)), )?; - // Every owner's partition was read once, at the watermark it served. + // Every owner's partition was read once, at the versions it reported. let mut reads = ShardReadLog::new(); for sr in &results { if let Some(target) = targets.iter().find(|t| t.node_id == sr.node_id) { - reads.note( - target.owned_vshards.iter().copied(), - sr.watermark_lsn, - target.node_id, - ); + reads.note(target.owned_vshards.iter().copied(), &sr.read_versions); } } reads.publish( - state, tenant_id, database_id, Some(qualified(database_id, ¶ms.collection)), diff --git a/nodedb/src/control/server/graph_dispatch/bsp_wcc/scatter.rs b/nodedb/src/control/server/graph_dispatch/bsp_wcc/scatter.rs index 719dd50fa..0231c040e 100644 --- a/nodedb/src/control/server/graph_dispatch/bsp_wcc/scatter.rs +++ b/nodedb/src/control/server/graph_dispatch/bsp_wcc/scatter.rs @@ -28,8 +28,8 @@ use nodedb_physical::physical_plan::{GraphOp, WccSuperstepPlan, WccSuperstepResu pub(super) struct ShardWccResult { pub(super) node_id: u64, pub(super) result: WccSuperstepResult, - /// The highest watermark the node's cores served the round at. - pub(super) watermark_lsn: crate::types::Lsn, + /// The versions the node's cores reported for the round. + pub(super) read_versions: crate::types::ReadVersions, } /// The inputs of one WCC round. @@ -100,7 +100,7 @@ pub(super) async fn scatter_wcc_round( Ok::(ShardWccResult { node_id, result, - watermark_lsn: read.watermark_lsn, + read_versions: read.read_versions, }) }) }); diff --git a/nodedb/src/control/server/graph_dispatch/cluster_resolve.rs b/nodedb/src/control/server/graph_dispatch/cluster_resolve.rs index 5ce92b04c..b75c79d91 100644 --- a/nodedb/src/control/server/graph_dispatch/cluster_resolve.rs +++ b/nodedb/src/control/server/graph_dispatch/cluster_resolve.rs @@ -31,15 +31,15 @@ pub(in crate::control::server::graph_dispatch) struct DispatchSuperstepParams<'a pub(in crate::control::server::graph_dispatch) linearizable: bool, } -/// One owner node's answer to a graph superstep: its payload and the highest -/// watermark its cores served the plan at. +/// One owner node's answer to a graph superstep: its payload and the +/// versions its cores reported. pub(in crate::control::server::graph_dispatch) struct NodeRead { pub(in crate::control::server::graph_dispatch) payload: Payload, - pub(in crate::control::server::graph_dispatch) watermark_lsn: crate::types::Lsn, + pub(in crate::control::server::graph_dispatch) read_versions: crate::types::ReadVersions, } /// Dispatch a single already-built graph-superstep `plan` to one owner node and -/// return its node-level payload and served watermark. The LOCAL node fans the plan across all its +/// return its node-level payload and reported versions. The LOCAL node fans the plan across all its /// Data-Plane cores via `execute_plan_all_local_cores` (per-core results merged /// into one payload); a REMOTE node gets one `RouteDecision::Remote` dispatch via /// `dispatch_route`. An empty payload denotes a zero-vertex shard — the caller's @@ -81,7 +81,7 @@ pub(in crate::control::server::graph_dispatch) async fn dispatch_superstep_to_no .await?; Ok(NodeRead { payload: Payload::from_vec(node_result.payload), - watermark_lsn: node_result.watermark_lsn, + read_versions: node_result.read_versions, }) } else { // Remote node: one dispatch via the gateway. @@ -106,12 +106,7 @@ pub(in crate::control::server::graph_dispatch) async fn dispatch_superstep_to_no linearizable, }) .await?; - let watermark_lsn = outcome - .shard_watermarks - .iter() - .map(|(_, lsn)| *lsn) - .max() - .unwrap_or(crate::types::Lsn::ZERO); + let read_versions = outcome.read_versions; let payload = outcome .payloads .into_iter() @@ -122,7 +117,7 @@ pub(in crate::control::server::graph_dispatch) async fn dispatch_superstep_to_no })?; Ok(NodeRead { payload, - watermark_lsn, + read_versions, }) } } diff --git a/nodedb/src/control/server/graph_dispatch/helpers.rs b/nodedb/src/control/server/graph_dispatch/helpers.rs index 1f63085ec..37bff36be 100644 --- a/nodedb/src/control/server/graph_dispatch/helpers.rs +++ b/nodedb/src/control/server/graph_dispatch/helpers.rs @@ -23,7 +23,7 @@ pub(super) fn ok_response(payload: Vec) -> Response { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/graph_dispatch/hop.rs b/nodedb/src/control/server/graph_dispatch/hop.rs index ca7080fc0..19ad04d0a 100644 --- a/nodedb/src/control/server/graph_dispatch/hop.rs +++ b/nodedb/src/control/server/graph_dispatch/hop.rs @@ -80,8 +80,8 @@ pub(super) struct HopOutput { /// Deduplicated destination node IDs after merging local + remote /// expansion. Feeds the next frontier. pub merged_destinations: Vec, - /// The key vShard of every frontier node, at the watermark its expansion - /// served. + /// The key vShard of every frontier node, at the version its expansion + /// reported. pub reads: ShardReadLog, } @@ -156,12 +156,14 @@ pub(super) async fn execute_neighbor_hop( // Single-node mode: no routing table — every frontier node is local. if shared.cluster_routing.is_none() { - let (rows, _watermark) = expand_local(shared, scope, frontier).await?; + let (rows, versions) = expand_local(shared, scope, frontier).await?; let merged = dedup_destinations(&rows); + let mut reads = ShardReadLog::new(); + reads.note(key_vshards(frontier), &versions); return Ok(HopOutput { rows, merged_destinations: merged, - reads: ShardReadLog::new(), + reads, }); } @@ -174,8 +176,8 @@ pub(super) async fn execute_neighbor_hop( let mut all_rows: Vec = if local_nodes.is_empty() { Vec::new() } else { - let (rows, watermark) = expand_local(shared, scope, &local_nodes).await?; - reads.note(key_vshards(&local_nodes), watermark, shared.node_id); + let (rows, versions) = expand_local(shared, scope, &local_nodes).await?; + reads.note(key_vshards(&local_nodes), &versions); rows }; @@ -307,12 +309,12 @@ impl ExpandScope<'_> { } /// Expand a locally-owned subset on all local Data-Plane cores. Returns the -/// crossed edges and the highest watermark a core served them at. +/// crossed edges and the versions the cores reported. async fn expand_local( shared: &SharedState, scope: ExpandScope<'_>, node_ids: &[String], -) -> crate::Result<(Vec, crate::types::Lsn)> { +) -> crate::Result<(Vec, crate::types::ReadVersions)> { let plan = scope.plan(node_ids.to_vec()); // A linearizable hop confirms the groups its plan reads first. if scope.linearizable { @@ -327,13 +329,13 @@ async fn expand_local( scope.txn_id, ) .await?; - Ok((decode_neighbor_rows(&resp.payload)?, resp.watermark_lsn)) + Ok((decode_neighbor_rows(&resp.payload)?, resp.read_versions)) } /// Expand the remote-owned subsets concurrently: ship a typed /// `NeighborsMulti` plan to each owning node via [`dispatch_route`] and /// decode every returned payload with the shared `{src,label,node}` decoder. -/// Notes each owner's vShard in `reads` at the watermark it served. +/// Notes each owner's vShard in `reads` at the version it reported. async fn expand_remote( shared: &SharedState, scope: ExpandScope<'_>, @@ -389,7 +391,7 @@ async fn expand_remote( linearizable, }) .await - .map(|outcome| (read_vshard, node_id, outcome)) + .map(|outcome| (read_vshard, outcome)) }) }); @@ -399,8 +401,8 @@ async fn expand_remote( for result in results { // A remote dispatch error is fatal: a dropped owner means a partial // reachable set. - let (read_vshard, node_id, outcome) = result?; - reads.note_leg([read_vshard], &outcome.shard_watermarks, node_id); + let (read_vshard, outcome) = result?; + reads.note([read_vshard], &outcome.read_versions); for payload in outcome.payloads { rows.extend(decode_neighbor_rows(&payload)?); } diff --git a/nodedb/src/control/server/graph_dispatch/keyed_read.rs b/nodedb/src/control/server/graph_dispatch/keyed_read.rs index d75ba2a56..d54974b01 100644 --- a/nodedb/src/control/server/graph_dispatch/keyed_read.rs +++ b/nodedb/src/control/server/graph_dispatch/keyed_read.rs @@ -85,7 +85,7 @@ pub async fn read_on_vshard( txn_id, ) .await?; - reads.note([vshard_id], response.watermark_lsn, state.node_id); + reads.note([vshard_id], &response.read_versions); response.payload } RouteDecision::Remote { @@ -113,7 +113,7 @@ pub async fn read_on_vshard( linearizable, }) .await?; - reads.note_leg([vshard_id], &outcome.shard_watermarks, node_id); + reads.note([vshard_id], &outcome.read_versions); let payloads = outcome.payloads; Payload::from_vec(match payloads.len() { 1 => payloads.into_iter().next().unwrap_or_default(), @@ -135,7 +135,7 @@ pub async fn read_on_vshard( } }; // The node's key vShard joins the transaction read-set. - reads.publish(state, tenant_id, database_id, collection); + reads.publish(tenant_id, database_id, collection); Ok(payload) } diff --git a/nodedb/src/control/server/graph_dispatch/match_broadcast.rs b/nodedb/src/control/server/graph_dispatch/match_broadcast.rs index 08d7d9ce0..96de6264e 100644 --- a/nodedb/src/control/server/graph_dispatch/match_broadcast.rs +++ b/nodedb/src/control/server/graph_dispatch/match_broadcast.rs @@ -32,7 +32,9 @@ use futures::future::join_all; use crate::bridge::envelope::{Payload, PhysicalPlan, Response}; -use crate::control::server::exchange::core_outcome::{classify_core_response, require_every_core}; +use crate::control::server::exchange::core_outcome::{ + classify_core_response, reported_versions, require_every_core, +}; use crate::control::server::exchange::gather::eager_dispatch_to_all_cores; use crate::control::server::payload_merge::{encode_msgpack_array, extract_msgpack_elements}; use crate::data::executor::handlers::graph_match::{ @@ -73,6 +75,8 @@ pub struct MatchBroadcastOutcome { pub partial: bool, /// The highest watermark any core served the MATCH at. pub watermark_lsn: crate::types::Lsn, + /// The versions every core reported, one per vShard it holds. + pub read_versions: crate::types::ReadVersions, } /// Locate a top-level map value by key in a msgpack map payload. @@ -265,6 +269,7 @@ pub async fn broadcast_match_to_all_cores( }); let results: Vec> = join_all(response_futures).await; + let read_versions = reported_versions(&results); // `NotFound` is an empty CSR slice on that core. Any other core error fails // the MATCH: rows from the surviving cores are not the full answer. let answered = require_every_core(results.into_iter().map(classify_core_response))?; @@ -302,6 +307,7 @@ pub async fn broadcast_match_to_all_cores( resume, partial, watermark_lsn, + read_versions, }) } diff --git a/nodedb/src/control/server/graph_dispatch/match_scatter/coord.rs b/nodedb/src/control/server/graph_dispatch/match_scatter/coord.rs index 01765db8d..901c0798a 100644 --- a/nodedb/src/control/server/graph_dispatch/match_scatter/coord.rs +++ b/nodedb/src/control/server/graph_dispatch/match_scatter/coord.rs @@ -214,7 +214,6 @@ pub async fn scatter_match( let rows_payload = dedup_and_encode(&coordinator.completed)?; // Every vShard round 0 read joins the transaction read-set. reads.publish( - state, tenant_id, database_id, pattern_collection(database_id, &query_bytes), diff --git a/nodedb/src/control/server/graph_dispatch/match_scatter/round_zero.rs b/nodedb/src/control/server/graph_dispatch/match_scatter/round_zero.rs index 4b6aba2c8..e134bf6fe 100644 --- a/nodedb/src/control/server/graph_dispatch/match_scatter/round_zero.rs +++ b/nodedb/src/control/server/graph_dispatch/match_scatter/round_zero.rs @@ -44,7 +44,7 @@ struct RoundZeroOwners { /// Every leg walks every vShard its node leads, so round 0 reads every vShard /// of the graph. A MATCH result depends on every one of them: an edge written /// to any vShard can add a match. The returned log notes each vShard under the -/// leg it was dispatched to, at the watermark that leg served, from the same +/// leg it was dispatched to, at the versions that leg reported, from the same /// routing snapshot that picked the legs. Later rounds read vShards round 0 /// already noted. pub(super) async fn scatter_round_zero( @@ -120,7 +120,7 @@ pub(super) async fn scatter_round_zero( }) .await?; let mut log = ShardReadLog::new(); - log.note_leg(leg_vshards, &outcome.shard_watermarks, node_id); + log.note(leg_vshards, &outcome.read_versions); Ok::<_, crate::Error>((collect_remote_envelopes(node_id, outcome.payloads)?, log)) }) }); @@ -132,7 +132,7 @@ pub(super) async fn scatter_round_zero( let mut out: Vec = Vec::new(); let mut log = ShardReadLog::new(); let local_outcome = local_outcome?; - log.note(local_vshards, local_outcome.watermark_lsn, state.node_id); + log.note(local_vshards, &local_outcome.read_versions); out.push(TaggedShardResult { emitting_node: state.node_id, rows: decode_rows(&local_outcome.rows_payload)?, diff --git a/nodedb/src/control/server/graph_dispatch/rag_fusion/coord.rs b/nodedb/src/control/server/graph_dispatch/rag_fusion/coord.rs index 54ab49b42..4c0f32f54 100644 --- a/nodedb/src/control/server/graph_dispatch/rag_fusion/coord.rs +++ b/nodedb/src/control/server/graph_dispatch/rag_fusion/coord.rs @@ -129,7 +129,7 @@ async fn run(state: &SharedState, scope: RagScope, plan: &PhysicalPlan) -> crate ) .await? else { - reads.publish(state, tenant_id, database_id, Some(qualified)); + reads.publish(tenant_id, database_id, Some(qualified)); return Ok(not_found_response()); }; let legs = exported.legs; @@ -270,7 +270,7 @@ async fn run(state: &SharedState, scope: RagScope, plan: &PhysicalPlan) -> crate detail: format!("{op_name} encode: {e}"), })?; // Every vShard the fusion read joins the transaction read-set. - reads.publish(state, tenant_id, database_id, Some(qualified)); + reads.publish(tenant_id, database_id, Some(qualified)); Ok(ok_payload_response(Payload::from_vec(payload))) } diff --git a/nodedb/src/control/server/graph_dispatch/rag_fusion/stages.rs b/nodedb/src/control/server/graph_dispatch/rag_fusion/stages.rs index 30bc7018a..1c08c512d 100644 --- a/nodedb/src/control/server/graph_dispatch/rag_fusion/stages.rs +++ b/nodedb/src/control/server/graph_dispatch/rag_fusion/stages.rs @@ -62,10 +62,6 @@ pub(super) async fn export_legs( } = scope; let shared = gateway_shared(state)?; let decision = resolve_live_decision(state, vshard); - let served_by = match decision { - crate::control::gateway::RouteDecision::Remote { node_id, .. } => node_id, - _ => state.node_id, - }; let route = TaskRoute { plan: with_stage(plan, RagStage::ExportLegs), decision, @@ -83,7 +79,7 @@ pub(super) async fn export_legs( linearizable, }) .await?; - reads.note_leg([vshard], &outcome.shard_watermarks, served_by); + reads.note([vshard], &outcome.read_versions); let watermark_lsn = outcome .shard_watermarks .iter() diff --git a/nodedb/src/control/server/graph_dispatch/shard_reads.rs b/nodedb/src/control/server/graph_dispatch/shard_reads.rs index e8fd86ed6..1d3917a34 100644 --- a/nodedb/src/control/server/graph_dispatch/shard_reads.rs +++ b/nodedb/src/control/server/graph_dispatch/shard_reads.rs @@ -3,35 +3,28 @@ //! The vShards a cross-shard graph read observed, for the transaction //! read-set. //! -//! A coordinator notes each vShard its legs read, at the watermark the -//! serving node reported, then publishes the log once the read finishes -//! (`session::graph_reads`). The request's protocol records the published +//! A coordinator notes each vShard its legs read, at the version the serving +//! core reported, then publishes the log once the read finishes +//! (`session::pending_shard_reads`). The request's protocol records the published //! reads into the transaction read-set, so commit validation checks every -//! vShard the read depended on. A read that runs on one node only (no -//! cluster) publishes nothing: its cores' watermarks sit in this node's WAL, -//! which single-shard SI already compares against. +//! vShard the read depended on. A version is the data-group log position of +//! the write that set it, the same on every replica, so a leg any replica +//! served compares with the vShard leader's versions. use std::collections::BTreeMap; -use crate::control::server::shared::session::graph_reads::{ - GraphShardReads, ShardObservation, note, +use nodedb_types::WriteVersion; + +use crate::control::server::shared::session::pending_shard_reads::{ + ShardObservation, ShardReads, note, }; -use crate::control::state::SharedState; -use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; +use crate::control::server::shared::session::read_set::EngineTag; +use crate::types::{DatabaseId, ReadVersions, TenantId, VShardId}; -/// Every vShard a graph read observed, each at its earliest served watermark -/// and with the node that served it. +/// Every vShard a graph read observed, each at its earliest observed version. #[derive(Debug, Default)] pub(crate) struct ShardReadLog { - served: BTreeMap, -} - -/// One vShard's observation: the earliest watermark, and the node whose WAL -/// numbers it (`0` once two nodes served the vShard). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct Served { - pub(crate) watermark: Lsn, - pub(crate) node: u64, + versions: BTreeMap, } impl ShardReadLog { @@ -39,85 +32,60 @@ impl ShardReadLog { Self::default() } - /// Note that `node` served `vshards` at `watermark`. A vShard read twice - /// keeps its earlier watermark: the first observation is the one the - /// result rests on. A vShard two nodes served has no one node whose - /// versions compare with the read, so its node becomes `0`. - pub(crate) fn note( - &mut self, - vshards: impl IntoIterator, - watermark: Lsn, - node: u64, - ) { + /// Note that a leg read `vshards` and reported `versions`. A vShard the + /// leg reported no version of had no write its serving core knew of, so + /// the leg observed it at `WriteVersion::ZERO`. A vShard read twice keeps + /// its earlier version: the first observation is the one the result rests + /// on. + pub(crate) fn note(&mut self, vshards: impl IntoIterator, versions: &ReadVersions) { for vshard in vshards { - self.served - .entry(vshard) - .and_modify(|seen| { - seen.watermark = seen.watermark.min(watermark); - if seen.node != node { - seen.node = 0; - } - }) - .or_insert(Served { watermark, node }); + let version = versions.of(VShardId::new(vshard)).unwrap_or_default(); + self.note_version(vshard, version); } } - /// Note one leg's response from `node`: `vshards` read at the highest - /// watermark the leg reported. - pub(crate) fn note_leg( - &mut self, - vshards: impl IntoIterator, - watermarks: &[(VShardId, Lsn)], - node: u64, - ) { - let watermark = watermarks - .iter() - .map(|(_, lsn)| *lsn) - .max() - .unwrap_or(Lsn::ZERO); - self.note(vshards, watermark, node); + fn note_version(&mut self, vshard: u32, version: WriteVersion) { + self.versions + .entry(vshard) + .and_modify(|seen| *seen = (*seen).min(version)) + .or_insert(version); } /// Fold `other` into this log. pub(crate) fn merge(&mut self, other: ShardReadLog) { - for (vshard, served) in other.served { - self.note([vshard], served.watermark, served.node); + for (vshard, version) in other.versions { + self.note_version(vshard, version); } } /// Hand the log to the running request for its transaction read-set. /// `collection` is the database-qualified collection the read scoped, or - /// `None` when it walked every collection. Without a cluster the log is - /// dropped (see the module doc). + /// `None` when it walked every collection. pub(crate) fn publish( self, - state: &SharedState, tenant_id: TenantId, database_id: DatabaseId, collection: Option, ) { - if state.cluster_routing.is_none() { - return; - } - note(GraphShardReads { + note(ShardReads { + engine: EngineTag::Graph, tenant_id, database_id, collection, shards: self - .served + .versions .into_iter() - .map(|(vshard, served)| ShardObservation { + .map(|(vshard, version)| ShardObservation { vshard: VShardId::new(vshard), - watermark: served.watermark, - node: served.node, + version, }) .collect(), }); } #[cfg(test)] - pub(crate) fn served(&self) -> &BTreeMap { - &self.served + pub(crate) fn versions(&self) -> &BTreeMap { + &self.versions } } @@ -140,52 +108,43 @@ pub(crate) fn key_vshards<'a>(nodes: impl IntoIterator) -> Ve mod tests { use super::*; - fn watermark(log: &ShardReadLog, vshard: u32) -> Option { - log.served().get(&vshard).map(|served| served.watermark) + fn at(index: u64) -> WriteVersion { + WriteVersion::logged(1, index) } - #[test] - fn a_vshard_read_twice_keeps_its_earliest_watermark() { - let mut log = ShardReadLog::new(); - log.note([3, 4], Lsn::new(20), 1); - log.note([3], Lsn::new(10), 1); - log.note([4], Lsn::new(30), 1); - assert_eq!(watermark(&log, 3), Some(Lsn::new(10))); - assert_eq!(watermark(&log, 4), Some(Lsn::new(20))); - assert_eq!(log.served().get(&3).map(|s| s.node), Some(1)); + fn reported(pairs: &[(u32, u64)]) -> ReadVersions { + let mut versions = ReadVersions::new(); + for (vshard, index) in pairs { + versions.note(VShardId::new(*vshard), at(*index)); + } + versions } #[test] - fn a_vshard_two_nodes_served_has_no_node() { + fn a_vshard_read_twice_keeps_its_earliest_version() { let mut log = ShardReadLog::new(); - log.note([3], Lsn::new(20), 1); - log.note([3], Lsn::new(30), 2); - assert_eq!(log.served().get(&3).map(|s| s.node), Some(0)); + log.note([3, 4], &reported(&[(3, 20), (4, 20)])); + log.note([3], &reported(&[(3, 10)])); + log.note([4], &reported(&[(4, 30)])); + assert_eq!(log.versions().get(&3), Some(&at(10))); + assert_eq!(log.versions().get(&4), Some(&at(20))); } #[test] - fn a_leg_is_noted_at_its_highest_reported_watermark() { + fn a_vshard_the_leg_reported_no_version_of_is_observed_at_zero() { let mut log = ShardReadLog::new(); - log.note_leg( - [7], - &[ - (VShardId::new(1), Lsn::new(5)), - (VShardId::new(2), Lsn::new(9)), - ], - 2, - ); - assert_eq!(watermark(&log, 7), Some(Lsn::new(9))); + log.note([7], &reported(&[(2, 9)])); + assert_eq!(log.versions().get(&7), Some(&WriteVersion::ZERO)); } #[test] - fn merging_keeps_the_earliest_watermark_per_vshard() { + fn merging_keeps_the_earliest_version_per_vshard() { let mut left = ShardReadLog::new(); - left.note([1], Lsn::new(8), 1); + left.note([1], &reported(&[(1, 8)])); let mut right = ShardReadLog::new(); - right.note([1], Lsn::new(4), 1); - right.note([2], Lsn::new(6), 1); + right.note([1, 2], &reported(&[(1, 4), (2, 6)])); left.merge(right); - assert_eq!(watermark(&left, 1), Some(Lsn::new(4))); - assert_eq!(watermark(&left, 2), Some(Lsn::new(6))); + assert_eq!(left.versions().get(&1), Some(&at(4))); + assert_eq!(left.versions().get(&2), Some(&at(6))); } } diff --git a/nodedb/src/control/server/graph_dispatch/shortest_path.rs b/nodedb/src/control/server/graph_dispatch/shortest_path.rs index 31019b246..16b89fe5a 100644 --- a/nodedb/src/control/server/graph_dispatch/shortest_path.rs +++ b/nodedb/src/control/server/graph_dispatch/shortest_path.rs @@ -109,7 +109,7 @@ pub async fn cross_core_shortest_path( &mut presence_reads, ) .await?; - presence_reads.publish(shared, tenant_id, database_id, None); + presence_reads.publish(tenant_id, database_id, None); if !present.contains(&src) || !present.contains(&dst) { return Ok(ok_response(encode_path::(&[])?)); } @@ -182,7 +182,7 @@ pub async fn cross_core_shortest_path( } // Every vShard the walk expanded joins the transaction read-set. - reads.publish(shared, tenant_id, database_id, collection); + reads.publish(tenant_id, database_id, collection); Ok(ok_response(encode_path(&path)?)) } diff --git a/nodedb/src/control/server/graph_dispatch/traverse_subgraph.rs b/nodedb/src/control/server/graph_dispatch/traverse_subgraph.rs index bc9037b5d..56ed05afe 100644 --- a/nodedb/src/control/server/graph_dispatch/traverse_subgraph.rs +++ b/nodedb/src/control/server/graph_dispatch/traverse_subgraph.rs @@ -210,7 +210,7 @@ pub(crate) async fn walk_subgraph( &mut presence_reads, ) .await?; - presence_reads.publish(shared, tenant_id, database_id, None); + presence_reads.publish(tenant_id, database_id, None); if !present.contains(&start) { return Ok(SubgraphWalk { node_order: Vec::new(), @@ -259,7 +259,7 @@ pub(crate) async fn walk_subgraph( } // Every vShard the walk expanded joins the transaction read-set. - reads.publish(shared, tenant_id, database_id, collection); + reads.publish(tenant_id, database_id, collection); Ok(SubgraphWalk { node_order, edges: edges.into_vec(), diff --git a/nodedb/src/control/server/graph_dispatch/whole_graph.rs b/nodedb/src/control/server/graph_dispatch/whole_graph.rs index 1cd21678b..75f68861f 100644 --- a/nodedb/src/control/server/graph_dispatch/whole_graph.rs +++ b/nodedb/src/control/server/graph_dispatch/whole_graph.rs @@ -67,6 +67,10 @@ pub async fn scatter_to_graph_owners( let node = execute_plan_all_local_cores(state, tenant_id, database_id, plan, TraceId::ZERO, None) .await?; + // This node holds every vShard, so the read observed all of them. + let mut reads = ShardReadLog::new(); + reads.note(0..VShardId::COUNT, &node.read_versions); + reads.publish(tenant_id, database_id, collection); return Ok(vec![Payload::from_vec(node.payload)]); } let targets = enumerate_shards(state)?.targets; @@ -93,17 +97,17 @@ pub async fn scatter_to_graph_owners( }, ) .await?; - Ok::<_, crate::Error>((target.owned_vshards, target.node_id, read)) + Ok::<_, crate::Error>((target.owned_vshards, read)) } }); let mut reads = ShardReadLog::new(); let mut payloads = Vec::new(); for leg in join_all(legs).await { - let (owned_vshards, node_id, read) = leg?; - reads.note(owned_vshards, read.watermark_lsn, node_id); + let (owned_vshards, read) = leg?; + reads.note(owned_vshards, &read.read_versions); payloads.push(read.payload); } - reads.publish(state, tenant_id, database_id, collection); + reads.publish(tenant_id, database_id, collection); Ok(payloads) } diff --git a/nodedb/src/control/server/http/routes/query/materialized/request.rs b/nodedb/src/control/server/http/routes/query/materialized/request.rs index 65bb7af40..7c8671318 100644 --- a/nodedb/src/control/server/http/routes/query/materialized/request.rs +++ b/nodedb/src/control/server/http/routes/query/materialized/request.rs @@ -153,6 +153,7 @@ pub async fn query( output_schema: Some(&output_schema), database_id, sequences: &sequences, + client_session: None, }; let routed = route_http_statement( &exec, diff --git a/nodedb/src/control/server/http/routes/query/ndjson.rs b/nodedb/src/control/server/http/routes/query/ndjson.rs index 0b0708b35..3f85e492b 100644 --- a/nodedb/src/control/server/http/routes/query/ndjson.rs +++ b/nodedb/src/control/server/http/routes/query/ndjson.rs @@ -143,6 +143,7 @@ pub async fn query_ndjson( output_schema: Some(&output_schema), database_id, sequences: &sequences, + client_session: None, }; let routed = route_http_statement( &exec, diff --git a/nodedb/src/control/server/http/routes/query_stream.rs b/nodedb/src/control/server/http/routes/query_stream.rs index 27cd25374..d1fff32f5 100644 --- a/nodedb/src/control/server/http/routes/query_stream.rs +++ b/nodedb/src/control/server/http/routes/query_stream.rs @@ -269,7 +269,7 @@ mod tests { Ok(RowBatch { payload: json_object_batch(start, n), watermark_lsn: Lsn::ZERO, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), }) } diff --git a/nodedb/src/control/server/http/routes/ws_rpc/execute_sql.rs b/nodedb/src/control/server/http/routes/ws_rpc/execute_sql.rs index aa7e286eb..874abe777 100644 --- a/nodedb/src/control/server/http/routes/ws_rpc/execute_sql.rs +++ b/nodedb/src/control/server/http/routes/ws_rpc/execute_sql.rs @@ -90,6 +90,7 @@ pub async fn execute_sql( output_schema: Some(&output_schema), database_id, sequences: &sequences, + client_session: None, }; let routed = { let _request = shared.tenant_request_guard(tenant_id); diff --git a/nodedb/src/control/server/native/dispatch/graph_match.rs b/nodedb/src/control/server/native/dispatch/graph_match.rs index fb70e0cea..b1ba9fbfd 100644 --- a/nodedb/src/control/server/native/dispatch/graph_match.rs +++ b/nodedb/src/control/server/native/dispatch/graph_match.rs @@ -110,9 +110,8 @@ pub(crate) async fn handle_graph_match( Err(error) => return error_to_native(seq, &error), }; - // A MATCH issued inside a native transaction records a collection-scoped - // predicate read at the shard's watermark, identical to every other read - // seam. Single-shard direct op means one watermark and one entry. + // A MATCH issued inside a native transaction records the versions its + // core reported, identical to every other read seam. if (response.status == Status::Ok || response.error_code.as_deref() == Some(&crate::bridge::envelope::ErrorCode::NotFound)) && ctx.sessions.transaction_state(ctx.peer_addr) @@ -125,11 +124,9 @@ pub(crate) async fn handle_graph_match( ctx.tenant_id(), crate::control::server::shared::session::ResponseReads { plan: &plan_for_response, - watermarks: &[(vshard_id, response.watermark_lsn)], - read_version_lsn: response.read_version_lsn, + read_versions: &response.read_versions, found: response.status == Status::Ok, distributed_reads: &[], - read_lsn_vshard: vshard_id, }, ) .await; @@ -167,7 +164,7 @@ pub(crate) async fn handle_graph_match( /// local MATCH yields once its envelope is unwrapped. A result the scatter /// cannot finish is refused with `54001`, never returned partial. The /// scatter notes every vShard it read, and the session loop records them into -/// the transaction read-set (`session::graph_reads`). +/// the transaction read-set (`session::pending_shard_reads`). async fn match_across_shards( ctx: &DispatchCtx<'_>, seq: u64, diff --git a/nodedb/src/control/server/native/dispatch/raw_dispatch.rs b/nodedb/src/control/server/native/dispatch/raw_dispatch.rs index d5c4e0ff3..93a95aab2 100644 --- a/nodedb/src/control/server/native/dispatch/raw_dispatch.rs +++ b/nodedb/src/control/server/native/dispatch/raw_dispatch.rs @@ -179,7 +179,7 @@ async fn dispatch_external_crdt_apply( watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: outcome.write_version, + read_versions: outcome.write_versions, write_set: Vec::new(), }) } diff --git a/nodedb/src/control/server/native/dispatch/single_task.rs b/nodedb/src/control/server/native/dispatch/single_task.rs index 86f88cd69..ba28d0ed5 100644 --- a/nodedb/src/control/server/native/dispatch/single_task.rs +++ b/nodedb/src/control/server/native/dispatch/single_task.rs @@ -123,7 +123,7 @@ pub(super) async fn dispatch_single_task( watermark_lsn: Lsn::new(0), error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }; return data_plane_response_to_native(ctx, seq, &plan_for_staged_response, &synthetic); @@ -178,7 +178,7 @@ pub(super) async fn dispatch_single_task( } let plan_for_response = task.plan.clone(); - let task_vshard = task.vshard_id; + let task_database_id = task.database_id; match dispatch_authorized_single_task( ctx, task.tenant_id, @@ -189,6 +189,15 @@ pub(super) async fn dispatch_single_task( .await { Ok(resp) => { + // A committed write floors the session's later reads at the + // versions it stamped, as on every other protocol. + ctx.sessions.note_own_write_response( + ctx.peer_addr, + task_database_id, + ctx.tenant_id(), + &plan_for_response, + &resp, + ); // Track direct-op reads, including NotFound phantom observations, // identically to native SQL and pgwire conflict detection. let records_read = resp.status == Status::Ok @@ -205,11 +214,9 @@ pub(super) async fn dispatch_single_task( ctx.tenant_id(), crate::control::server::shared::session::ResponseReads { plan: &plan_for_response, - watermarks: &[(task_vshard, resp.watermark_lsn)], - read_version_lsn: resp.read_version_lsn, + read_versions: &resp.read_versions, found: resp.status == Status::Ok, distributed_reads: &[], - read_lsn_vshard: task_vshard, }, ) .await; diff --git a/nodedb/src/control/server/native/dispatch/sql_loop.rs b/nodedb/src/control/server/native/dispatch/sql_loop.rs index cb5836f77..58ca09b2a 100644 --- a/nodedb/src/control/server/native/dispatch/sql_loop.rs +++ b/nodedb/src/control/server/native/dispatch/sql_loop.rs @@ -71,6 +71,10 @@ pub(super) async fn run_dispatch_loop( output_schema, database_id, sequences: &sequences, + client_session: Some(DmlTxnCtx { + sessions: ctx.sessions, + session_id: ctx.peer_addr.into(), + }), }; let result = statement_exec::run_statement_loop( &exec, @@ -107,6 +111,10 @@ pub(super) async fn run_implicit_statement( output_schema, database_id, sequences: &sequences, + client_session: Some(DmlTxnCtx { + sessions: ctx.sessions, + session_id: ctx.peer_addr.into(), + }), }; let client = DmlTxnCtx { sessions: ctx.sessions, diff --git a/nodedb/src/control/server/native/dispatch/sql_planned.rs b/nodedb/src/control/server/native/dispatch/sql_planned.rs index 93b0d7522..d6ef1ed56 100644 --- a/nodedb/src/control/server/native/dispatch/sql_planned.rs +++ b/nodedb/src/control/server/native/dispatch/sql_planned.rs @@ -138,6 +138,10 @@ pub(super) async fn execute_planned( output_schema: Some(&output_schema), database_id, sequences: &sequences, + client_session: Some(DmlTxnCtx { + sessions: ctx.sessions, + session_id: ctx.peer_addr.into(), + }), }; let routed = route_atomic_statement( &exec, diff --git a/nodedb/src/control/server/native/session/run.rs b/nodedb/src/control/server/native/session/run.rs index 34edd61b6..624a3ee33 100644 --- a/nodedb/src/control/server/native/session/run.rs +++ b/nodedb/src/control/server/native/session/run.rs @@ -300,9 +300,9 @@ impl NativeSession { // Crash-injection coverage verifies that a panic after a request // mutates transaction state still runs detached connection cleanup. crate::fail_point!("native_session::after_request"); - // Cross-shard graph reads the request made join the transaction's + // Cross-shard reads the request made join the transaction's // read-set. - crate::control::server::shared::session::graph_reads::record_pending( + crate::control::server::shared::session::pending_shard_reads::record_pending( &self.sessions, self.peer_addr.into(), ); diff --git a/nodedb/src/control/server/native/session/session_stream.rs b/nodedb/src/control/server/native/session/session_stream.rs index e3157af60..d3844f4ea 100644 --- a/nodedb/src/control/server/native/session/session_stream.rs +++ b/nodedb/src/control/server/native/session/session_stream.rs @@ -218,7 +218,7 @@ mod tests { Ok(RowBatch { payload: json_batch(start, n), watermark_lsn: Lsn::ZERO, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), }) } @@ -352,7 +352,7 @@ mod tests { vec![Ok(RowBatch { payload: msgpack_empty_batch(3), watermark_lsn: Lsn::ZERO, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), })], usize::MAX, ) diff --git a/nodedb/src/control/server/pgwire/catalog/dropped_collections.rs b/nodedb/src/control/server/pgwire/catalog/dropped_collections.rs index 2dfaf47fb..a54f73297 100644 --- a/nodedb/src/control/server/pgwire/catalog/dropped_collections.rs +++ b/nodedb/src/control/server/pgwire/catalog/dropped_collections.rs @@ -120,6 +120,7 @@ async fn query_collection_size( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/control/server/pgwire/handler/core.rs b/nodedb/src/control/server/pgwire/handler/core.rs index 1568f7ad1..3e2f962a2 100644 --- a/nodedb/src/control/server/pgwire/handler/core.rs +++ b/nodedb/src/control/server/pgwire/handler/core.rs @@ -199,9 +199,9 @@ impl ExtendedQueryHandler for NodeDbPgHandler { .send(PgWireBackendMessage::NoticeResponse(notice)) .await; } - // Cross-shard graph reads the statement made join the transaction's + // Cross-shard reads the statement made join the transaction's // read-set. - crate::control::server::shared::session::graph_reads::record_pending( + crate::control::server::shared::session::pending_shard_reads::record_pending( &self.sessions, session_id, ); diff --git a/nodedb/src/control/server/pgwire/handler/core/simple_query.rs b/nodedb/src/control/server/pgwire/handler/core/simple_query.rs index 7fafc80f8..25645ecf9 100644 --- a/nodedb/src/control/server/pgwire/handler/core/simple_query.rs +++ b/nodedb/src/control/server/pgwire/handler/core/simple_query.rs @@ -125,9 +125,9 @@ impl SimpleQueryHandler for NodeDbPgHandler { .send(PgWireBackendMessage::NoticeResponse(notice)) .await; } - // Cross-shard graph reads the query made join the transaction's + // Cross-shard reads the query made join the transaction's // read-set. - crate::control::server::shared::session::graph_reads::record_pending( + crate::control::server::shared::session::pending_shard_reads::record_pending( &self.sessions, session_id, ); diff --git a/nodedb/src/control/server/pgwire/handler/dispatch/entry.rs b/nodedb/src/control/server/pgwire/handler/dispatch/entry.rs index cd6f60c8c..77e1b3536 100644 --- a/nodedb/src/control/server/pgwire/handler/dispatch/entry.rs +++ b/nodedb/src/control/server/pgwire/handler/dispatch/entry.rs @@ -11,7 +11,6 @@ use std::sync::Arc; use crate::bridge::envelope::Response; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::exchange::resolve::DistributedReadCapture; -use crate::types::{Lsn, VShardId}; use nodedb_physical::physical_task::PhysicalTask; use super::super::core::NodeDbPgHandler; @@ -30,30 +29,27 @@ impl NodeDbPgHandler { identity: &AuthenticatedIdentity, linearizable: bool, ) -> crate::Result { - let mut shard_watermarks = Vec::new(); let mut distributed_reads = Vec::new(); self.dispatch_task_inner( task, user_id, identity, linearizable, - &mut shard_watermarks, &mut distributed_reads, ) .await } - /// Dispatch a task and return the response, per-shard watermark LSNs a fan - /// gather observed, and per-side read captures a shuffle JOIN produced. - /// Used by the transactional read-recording seam. - pub(in crate::control::server::pgwire::handler) async fn dispatch_authorized_task_with_watermarks( + /// Dispatch a task and return the response and the per-side read captures + /// a distributed read produced. Used by the transactional read-recording + /// seam. + pub(in crate::control::server::pgwire::handler) async fn dispatch_authorized_task_with_captures( &self, task: PhysicalTask, user_id: Option>, identity: &AuthenticatedIdentity, linearizable: bool, - ) -> crate::Result<(Response, Vec<(VShardId, Lsn)>, Vec)> { - let mut shard_watermarks = Vec::new(); + ) -> crate::Result<(Response, Vec)> { let mut distributed_reads = Vec::new(); let resp = self .dispatch_task_inner( @@ -61,10 +57,9 @@ impl NodeDbPgHandler { user_id, identity, linearizable, - &mut shard_watermarks, &mut distributed_reads, ) .await?; - Ok((resp, shard_watermarks, distributed_reads)) + Ok((resp, distributed_reads)) } } diff --git a/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs b/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs index 7fb9e7638..2c9237cdd 100644 --- a/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs +++ b/nodedb/src/control/server/pgwire/handler/dispatch/replicated.rs @@ -28,9 +28,9 @@ impl NodeDbPgHandler { let ReplicatedWrite { entry, proposer } = args; let request_id = self.next_request_id(); - // `write_version` is the post-write `coll_write_lsn`, surfaced so the session - // can floor a later read-set at it (read-your-writes for cross-shard OCC). - let (payload, write_version) = crate::control::wal_replication::propose_replicated_entry( + // `write_versions` are the versions the write stamped. The session + // floors its later reads of the written vShards at them. + let (payload, write_versions) = crate::control::wal_replication::propose_replicated_entry( &self.state, proposer, entry, @@ -44,11 +44,11 @@ impl NodeDbPgHandler { attempt: 1, partial: false, payload: payload.into(), - // Authoritative participant WAL LSN — CDC ordering must use it, not zero. - watermark_lsn: write_version, + // A write carries no read watermark. + watermark_lsn: crate::types::Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: write_version, + read_versions: write_versions, write_set: Vec::new(), }; diff --git a/nodedb/src/control/server/pgwire/handler/dispatch/routing.rs b/nodedb/src/control/server/pgwire/handler/dispatch/routing.rs index 0cb929392..5cd4c48da 100644 --- a/nodedb/src/control/server/pgwire/handler/dispatch/routing.rs +++ b/nodedb/src/control/server/pgwire/handler/dispatch/routing.rs @@ -15,7 +15,7 @@ use crate::control::server::exchange::resolve::{ DistributedReadCapture, Resolved, resolve_and_materialize, }; use crate::control::server::shared::write_admission::plan_is_write; -use crate::types::{Lsn, ReadConsistency, TraceId, VShardId}; +use crate::types::{ReadConsistency, TraceId}; use nodedb_physical::physical_task::PhysicalTask; use super::super::core::NodeDbPgHandler; @@ -29,7 +29,6 @@ impl NodeDbPgHandler { user_id: Option>, identity: &AuthenticatedIdentity, linearizable: bool, - shard_watermarks: &mut Vec<(VShardId, Lsn)>, distributed_reads: &mut Vec, ) -> crate::Result { use crate::control::security::identity::{Permission, required_permission}; @@ -167,8 +166,7 @@ impl NodeDbPgHandler { linearizable, }; match resolve_and_materialize(&self.state, identity, task.plan, scope).await? { - Resolved::Gathered(resp, wms, caps) => { - *shard_watermarks = wms; + Resolved::Gathered(resp, _watermarks, caps) => { *distributed_reads = caps; return Ok(resp); } @@ -243,11 +241,7 @@ impl NodeDbPgHandler { txn_id: checked.txn_id(), linearizable, }; - let outcome = gateway.execute_with_watermarks(&ctx, checked).await; - if let Ok((_, wms, _)) = &outcome { - *shard_watermarks = wms.clone(); - } - return owner_response(outcome); + return owner_response(gateway.execute_outcome(&ctx, checked).await); } } } diff --git a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs index 186d34270..36748de24 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs @@ -253,7 +253,6 @@ impl NodeDbPgHandler { let tag_role = statement_tag.role_of(&task.plan); let resp_post_set_op = task.post_set_op; let task_database_id = task.database_id; - let task_vshard = task.vshard_id; let plan_for_response = task.plan.clone(); // Extracted from the clone above, before dispatch — metering // needs the collection/engine shape after this task's dispatch @@ -348,8 +347,8 @@ impl NodeDbPgHandler { let user_id: Option> = Some(std::sync::Arc::from(identity.username.as_str())); let linearizable = strong_reads && !plan_is_write(&task.plan); - let (resp, shard_watermarks, distributed_reads) = self - .dispatch_authorized_task_with_watermarks(task, user_id, identity, linearizable) + let (resp, distributed_reads) = self + .dispatch_authorized_task_with_captures(task, user_id, identity, linearizable) .await .map_err(|e| { let (severity, code, message) = error_to_sqlstate(&e); @@ -366,11 +365,9 @@ impl NodeDbPgHandler { client_session: session_id, txn, plan: &plan_for_response, - vshard: task_vshard, database_id: task_database_id, }, &resp, - shard_watermarks, &distributed_reads, ) .await; @@ -488,7 +485,7 @@ fn staged_rows_response(rows: Vec) -> crate::bridge::envelope::Response { watermark_lsn: crate::types::Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/tracking.rs b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/tracking.rs index 3a63315aa..e79dae379 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/tracking.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/tracking.rs @@ -4,13 +4,12 @@ //! validates at COMMIT, and the session's own committed write version. use crate::bridge::envelope::{ErrorCode, PhysicalPlan, Response, Status}; -use crate::control::security::identity::{AuthenticatedIdentity, Permission, required_permission}; +use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::exchange::resolve::DistributedReadCapture; -use crate::control::server::shared::plan_util::extract_collection; use crate::control::server::shared::session::{ DmlTxnCtx, ResponseReads, SessionId, record_reads_for_response, }; -use crate::types::{DatabaseId, Lsn, VShardId}; +use crate::types::DatabaseId; use super::super::super::core::NodeDbPgHandler; @@ -23,29 +22,25 @@ pub(super) struct DispatchedTask<'a> { /// COMMIT. `None` for an autocommit statement. pub txn: Option<&'a DmlTxnCtx<'a>>, pub plan: &'a PhysicalPlan, - pub vshard: VShardId, pub database_id: DatabaseId, } impl NodeDbPgHandler { /// Record what one dispatched task observed and wrote. /// - /// A read in a transaction joins the transaction's read set, for - /// snapshot-isolation and cross-shard conflict checks. An absent-key - /// point read (a `NotFound` from the Data Plane) joins it too: a "not - /// found" is a validatable phantom observation. A genuine dispatch - /// failure records nothing. + /// A read in a transaction joins the transaction's read set, for commit + /// conflict checks. An absent-key point read (a `NotFound` from the Data + /// Plane) joins it too: a "not found" is a validatable phantom + /// observation. A genuine dispatch failure records nothing. /// - /// A successful write records its committed per-collection version on - /// the client's session, so a later transaction's read-set capture is - /// floored at it (the read-your-writes floor for cross-shard OCC). An - /// autocommit write floors a later transaction's read too, so this - /// records outside a transaction as well. + /// A successful write records the versions it stamped on the client's + /// session, so a later transaction's read-set capture is floored at them + /// (the read-your-writes floor). An autocommit write floors a later + /// transaction's read too, so this records outside a transaction as well. pub(super) async fn track_dispatched_task( &self, task: DispatchedTask<'_>, resp: &Response, - shard_watermarks: Vec<(VShardId, Lsn)>, distributed_reads: &[DistributedReadCapture], ) { let DispatchedTask { @@ -53,17 +48,11 @@ impl NodeDbPgHandler { client_session, txn, plan, - vshard, database_id, } = task; let records_read = resp.status == Status::Ok || resp.error_code.as_deref() == Some(&ErrorCode::NotFound); if records_read && let Some(txn) = txn { - let watermarks = if shard_watermarks.is_empty() { - vec![(vshard, resp.watermark_lsn)] - } else { - shard_watermarks - }; record_reads_for_response( &self.state, txn.sessions, @@ -71,28 +60,20 @@ impl NodeDbPgHandler { identity.tenant_id, ResponseReads { plan, - watermarks: &watermarks, - read_version_lsn: resp.read_version_lsn, + read_versions: &resp.read_versions, found: resp.status == Status::Ok, distributed_reads, - read_lsn_vshard: vshard, }, ) .await; } - if resp.status == Status::Ok - && resp.read_version_lsn > Lsn::ZERO - && matches!(required_permission(plan), Permission::Write) - && let Some(collection) = extract_collection(plan) - { - self.sessions.note_own_write( - client_session, - database_id, - identity.tenant_id, - collection, - resp.read_version_lsn, - ); - } + self.sessions.note_own_write_response( + client_session, + database_id, + identity.tenant_id, + plan, + resp, + ); } } diff --git a/nodedb/src/control/server/result_stream.rs b/nodedb/src/control/server/result_stream.rs index 28dcfef9e..95d111e65 100644 --- a/nodedb/src/control/server/result_stream.rs +++ b/nodedb/src/control/server/result_stream.rs @@ -24,16 +24,16 @@ use crate::types::Lsn; /// /// `payload` is a standalone msgpack array (the exact bytes produced by a single /// Data-Plane scan chunk); `watermark_lsn` is that frame's read watermark and -/// `read_version_lsn` its per-collection read version. +/// `read_versions` its read versions. pub struct RowBatch { /// Standalone msgpack array of row elements for this frame. pub payload: Vec, /// Watermark LSN reported by the Data Plane for this frame. pub watermark_lsn: Lsn, - /// Per-collection read-version LSN for this frame (the scanned collection's - /// `coll_write_lsn` at read time) — the sound comparand for cross-shard OCC - /// read validation, distinct from the core-global `watermark_lsn`. - pub read_version_lsn: Lsn, + /// The read versions this frame observed, one per vShard: the comparand + /// for cross-shard OCC read validation, distinct from the core-global + /// `watermark_lsn`. + pub read_versions: crate::types::ReadVersions, } /// A pinned, boxed stream of [`RowBatch`]es, fallible per item. @@ -103,7 +103,7 @@ pub(crate) fn stream_response_channel( yield RowBatch { payload: resp.payload.to_vec(), watermark_lsn: resp.watermark_lsn, - read_version_lsn: resp.read_version_lsn, + read_versions: resp.read_versions.clone(), }; if is_terminal { return; @@ -157,7 +157,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -172,7 +172,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -187,7 +187,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -202,7 +202,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: Some(Box::new(code)), stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/shared/clone_read/dispatch.rs b/nodedb/src/control/server/shared/clone_read/dispatch.rs index 191729ef4..61b445050 100644 --- a/nodedb/src/control/server/shared/clone_read/dispatch.rs +++ b/nodedb/src/control/server/shared/clone_read/dispatch.rs @@ -209,7 +209,7 @@ pub(super) fn empty_response(state: &SharedState) -> Response { watermark_lsn: crate::types::Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/shared/clone_write/probes.rs b/nodedb/src/control/server/shared/clone_write/probes.rs index de59fdc6a..341083029 100644 --- a/nodedb/src/control/server/shared/clone_write/probes.rs +++ b/nodedb/src/control/server/shared/clone_write/probes.rs @@ -290,6 +290,7 @@ pub(super) async fn dispatch_data_plane_raw( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/control/server/shared/clone_write/util.rs b/nodedb/src/control/server/shared/clone_write/util.rs index 208eb6b65..c00b961a9 100644 --- a/nodedb/src/control/server/shared/clone_write/util.rs +++ b/nodedb/src/control/server/shared/clone_write/util.rs @@ -34,7 +34,7 @@ pub(super) fn synthetic_affected_response( watermark_lsn, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/purge/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/purge/dispatch.rs index bd8222ede..e07ee1859 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/purge/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/purge/dispatch.rs @@ -85,6 +85,7 @@ pub async fn dispatch_unregister_collection( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs index 4d2f91726..d0f10a046 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/kv_sorted_index/dispatch.rs @@ -156,7 +156,7 @@ fn verdict_response(code: ErrorCode) -> Response { watermark_lsn: crate::types::Lsn::ZERO, error_code: Some(Box::new(code)), stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/shared/ddl/neutral/maintenance/distributed.rs b/nodedb/src/control/server/shared/ddl/neutral/maintenance/distributed.rs index 20f4d0084..7cf174703 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/maintenance/distributed.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/maintenance/distributed.rs @@ -49,6 +49,7 @@ pub fn dispatch_maintenance_to_all_cores( wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/control/server/shared/ddl/neutral/rate_gate.rs b/nodedb/src/control/server/shared/ddl/neutral/rate_gate.rs index 0690c931f..abbabf120 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/rate_gate.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/rate_gate.rs @@ -432,7 +432,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/shared/ddl/neutral/router/string_engine_ops.rs b/nodedb/src/control/server/shared/ddl/neutral/router/string_engine_ops.rs index 52653aa95..9d0c904fe 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/router/string_engine_ops.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/router/string_engine_ops.rs @@ -368,9 +368,7 @@ mod tests { }; assert!(!crdt_apply_forbidden_in_transaction(&ctx)); - sessions - .begin(addr, crate::types::Lsn::new(1), 0) - .expect("begin"); + sessions.begin(addr, 0).expect("begin"); assert!(crdt_apply_forbidden_in_transaction(&ctx)); sessions.fail_transaction(addr); assert!(crdt_apply_forbidden_in_transaction(&ctx)); diff --git a/nodedb/src/control/server/shared/ddl/neutral/router/string_versioning.rs b/nodedb/src/control/server/shared/ddl/neutral/router/string_versioning.rs index 1d5334675..bd764f2e4 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/router/string_versioning.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/router/string_versioning.rs @@ -286,9 +286,7 @@ mod tests { session_id: SessionId::from(&addr), }; assert!(!restore_forbidden_in_transaction(&ctx)); - sessions - .begin(addr, crate::types::Lsn::new(1), 0) - .expect("begin"); + sessions.begin(addr, 0).expect("begin"); assert!(restore_forbidden_in_transaction(&ctx)); sessions.fail_transaction(addr); assert!(restore_forbidden_in_transaction(&ctx)); diff --git a/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs b/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs index 4f992a398..ae7ea0dba 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/version_history/restore.rs @@ -146,7 +146,7 @@ struct RestoreDeltaParams<'a> { async fn persist_restore_delta( state: &SharedState, params: RestoreDeltaParams<'_>, -) -> crate::Result> { +) -> crate::Result { let RestoreDeltaParams { tenant_id, database_id, @@ -182,7 +182,7 @@ async fn persist_restore_delta( }, ) .await?; - Ok((outcome.write_version != crate::types::Lsn::ZERO).then_some(outcome.write_version)) + Ok(outcome.write_versions) } /// Parse: RESTORE collection SET VERSION = 'checkpoint' WHERE id = 'doc-id' @@ -275,7 +275,7 @@ mod tests { /// A Data Plane core for a one-node cluster that answers the restore /// admission: the generate returns `delta`, the preview returns a result - /// with `digest`, and the fenced apply echoes its WAL LSN. It answers + /// with `digest`, and the fenced apply echoes its write version. It answers /// every other request, the cluster's Raft applies included, with `Ok`. /// It records each restore step it sees, marking a step whose delta or /// digest differs from the expected one. @@ -295,18 +295,32 @@ mod tests { while let Ok(request) = side.request_rx.try_pop() { let request = request.inner; let request_id = request.request_id; - let wal_lsn = request.wal_lsn.unwrap_or(Lsn::ZERO); - let (step, payload, read_version_lsn) = match request.plan { - PhysicalPlan::Crdt(CrdtOp::RestoreToVersion { .. }) => { - (Some("generate"), core.delta.clone(), Lsn::ZERO) - } + let applied_versions = crate::types::ReadVersions::single( + request.vshard_id, + request + .entry_version + .unwrap_or(nodedb_types::WriteVersion::local_after( + nodedb_types::WriteVersion::ZERO, + request.wal_lsn.unwrap_or(Lsn::ZERO).as_u64(), + )), + ); + let (step, payload, read_versions) = match request.plan { + PhysicalPlan::Crdt(CrdtOp::RestoreToVersion { .. }) => ( + Some("generate"), + core.delta.clone(), + crate::types::ReadVersions::new(), + ), PhysicalPlan::Crdt(CrdtOp::PreviewApply { delta, .. }) => { let step = if delta == core.delta { "preview" } else { "preview: wrong delta" }; - (Some(step), preview.clone(), Lsn::ZERO) + ( + Some(step), + preview.clone(), + crate::types::ReadVersions::new(), + ) } PhysicalPlan::Crdt(CrdtOp::Apply { delta, @@ -320,9 +334,9 @@ mod tests { } else { "apply: wrong delta or digest" }; - (Some(step), Vec::new(), wal_lsn) + (Some(step), Vec::new(), applied_versions) } - _ => (None, Vec::new(), Lsn::ZERO), + _ => (None, Vec::new(), crate::types::ReadVersions::new()), }; if let Some(step) = step { core.seen @@ -341,7 +355,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn, + read_versions, write_set: Vec::new(), }, }) @@ -424,15 +438,15 @@ mod tests { ) .await .expect("persist restore delta"); - let lsn = outcome.map(|outcome| outcome.write_version); + let versions = outcome.map(|outcome| outcome.write_versions); assert_eq!( core.seen(), ["generate", "preview", "apply"], "restore admission must generate, preview, then apply" ); assert!( - lsn.is_some(), - "a restore must allocate and return a durable WAL LSN" + versions.is_some_and(|versions| !versions.is_empty()), + "a restore must return the version its apply stamped" ); state.wal.sync().expect("sync wal"); diff --git a/nodedb/src/control/server/shared/ddl/sync_dispatch/dispatch.rs b/nodedb/src/control/server/shared/ddl/sync_dispatch/dispatch.rs index f1998526b..09cf73d83 100644 --- a/nodedb/src/control/server/shared/ddl/sync_dispatch/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/sync_dispatch/dispatch.rs @@ -200,6 +200,7 @@ async fn dispatch_plan(state: &SharedState, dispatch: PlanDispatch) -> crate::Re wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/control/server/shared/ddl/user_dispatch.rs b/nodedb/src/control/server/shared/ddl/user_dispatch.rs index 1cf3bd18d..a507c446d 100644 --- a/nodedb/src/control/server/shared/ddl/user_dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/user_dispatch.rs @@ -493,7 +493,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) diff --git a/nodedb/src/control/server/shared/metering.rs b/nodedb/src/control/server/shared/metering.rs index d0da235bc..c1a151d5e 100644 --- a/nodedb/src/control/server/shared/metering.rs +++ b/nodedb/src/control/server/shared/metering.rs @@ -963,7 +963,7 @@ mod tests { watermark_lsn: crate::types::Lsn::new(0), error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/shared/response_payload.rs b/nodedb/src/control/server/shared/response_payload.rs index 53ce1feb5..905ba3f42 100644 --- a/nodedb/src/control/server/shared/response_payload.rs +++ b/nodedb/src/control/server/shared/response_payload.rs @@ -45,7 +45,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: code.map(Box::new), stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/shared/session/commit/conflict.rs b/nodedb/src/control/server/shared/session/commit/conflict.rs deleted file mode 100644 index 8521a5ffc..000000000 --- a/nodedb/src/control/server/shared/session/commit/conflict.rs +++ /dev/null @@ -1,56 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Snapshot-isolation write-conflict validation for a single-shard COMMIT. - -use crate::control::state::SharedState; - -use super::super::connection::SessionId; -use super::super::outcome::{AbortReason, CommitOutcome}; -use super::super::read_set::ReadSetEntry; -use super::super::store::SessionStore; - -/// Snapshot-isolation write-conflict check for a single-shard interactive -/// COMMIT. If any read key's collection advanced past both the read LSN and the -/// transaction snapshot LSN — and the transaction did not write that collection -/// itself (read-your-own-write is excluded) — the WAL moved under the reader: -/// records the read-set hot-key aborts and returns a serialization abort. The -/// caller owns the session rollback (via `release_and_rollback`) so it can -/// first release the transaction's read reservations while the reservation owner -/// is still set — the rollback clears it. Returns `None` when there is no -/// conflict (or no snapshot, i.e. not in a transaction). -/// -/// This is a single-shard validation: it compares against the global WAL -/// `next_lsn`, so it is only sound for a transaction whose participants are one -/// shard, and is run exclusively on the `SingleShard` / read-only paths. -pub(super) fn si_conflict_abort( - sessions: &SessionStore, - session_id: SessionId, - state: &SharedState, - read_set: &[ReadSetEntry], - written_collections: &std::collections::HashSet, -) -> Option { - let snapshot_lsn = sessions.snapshot_lsn(session_id)?; - let current_lsn = state.wal.next_lsn(); - let current = crate::types::Lsn::new(current_lsn.as_u64().saturating_sub(1)); - for entry in read_set { - // A homed read's watermark can come from another node's WAL, so this - // node's WAL cannot judge it. `homed_reads` validates it on its home. - if entry.home.is_some() { - continue; - } - let collection = &entry.collection; - let read_lsn = entry.read_lsn; - if written_collections.contains(collection) { - continue; - } - if current > read_lsn && current > snapshot_lsn { - // WAL advanced past what we read — concurrent write detected. The - // caller releases reservations and rolls the session back. - super::super::hot_key::record_read_set_aborts(state, read_set); - return Some(CommitOutcome::Aborted { - reason: AbortReason::Serialization, - }); - } - } - None -} diff --git a/nodedb/src/control/server/shared/session/commit/mod.rs b/nodedb/src/control/server/shared/session/commit/mod.rs index 13b97e7cc..26eea7e14 100644 --- a/nodedb/src/control/server/shared/session/commit/mod.rs +++ b/nodedb/src/control/server/shared/session/commit/mod.rs @@ -2,9 +2,9 @@ //! Protocol-neutral COMMIT orchestration shared by pgwire and native sessions. -pub mod conflict; -mod homed_reads; pub mod metering; +mod read_split; +mod read_validation; pub mod restart_identity; pub mod run; pub mod single_shard; diff --git a/nodedb/src/control/server/shared/session/commit/read_split.rs b/nodedb/src/control/server/shared/session/commit/read_split.rs new file mode 100644 index 000000000..69d2db4f5 --- /dev/null +++ b/nodedb/src/control/server/shared/session/commit/read_split.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Which reads of a COMMIT enlist Calvin participants, and which this node +//! checks itself before dispatch. +//! +//! A homed read is one vShard of a cross-shard graph or array read. A +//! whole-graph read homes on every vShard. On a node with no cluster routing, +//! every vShard is local, so this node checks such reads against its own +//! cores and they enlist no participant. A single-node transaction that makes +//! a cross-shard read and writes one vShard then keeps its single-vShard +//! commit. In a cluster, every read enlists its vShard, so a participant +//! checks it inside the Calvin barrier. + +use crate::control::state::SharedState; + +use super::super::connection::SessionId; +use super::super::outcome::CommitOutcome; +use super::super::read_set::ReadSetEntry; +use super::super::store::SessionStore; +use super::read_validation::stale_read_abort; + +/// A COMMIT's read set, split by who validates each read. +pub(super) struct SplitReads<'a> { + all: &'a [ReadSetEntry], + /// `(enlisted, checked_here)` when this node checks some reads itself. + local: Option<(Vec, Vec)>, +} + +impl<'a> SplitReads<'a> { + pub(super) fn of(state: &SharedState, all: &'a [ReadSetEntry]) -> Self { + let local = (state.cluster_routing.is_none() && all.iter().any(|read| read.home.is_some())) + .then(|| { + let (here, enlisted): (Vec<_>, Vec<_>) = + all.iter().cloned().partition(|read| read.home.is_some()); + (enlisted, here) + }); + Self { all, local } + } + + /// The reads that enlist their vShard as a participant. + pub(super) fn enlisted(&self) -> &[ReadSetEntry] { + match &self.local { + Some((enlisted, _)) => enlisted, + None => self.all, + } + } + + /// The reads this node checks before dispatch. + fn checked_here(&self) -> &[ReadSetEntry] { + match &self.local { + Some((_, here)) => here, + None => &[], + } + } + + /// Abort the transaction when a read this node checks itself is no + /// longer current. Releases the reservations and rolls the session back + /// before returning the outcome. `None` when every such read holds, or + /// there is none. + pub(super) async fn stale_here( + &self, + state: &SharedState, + sessions: &SessionStore, + session_id: SessionId, + written_collections: &std::collections::HashSet, + ) -> Option { + let here = self.checked_here(); + if here.is_empty() { + return None; + } + stale_read_abort(state, sessions, session_id, here, written_collections).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::server::shared::session::read_set::{EngineTag, ReadKey, ReadOrigin}; + use crate::types::{DatabaseId, TenantId, VShardId}; + + fn read(collection: &str, home: Option) -> ReadSetEntry { + ReadSetEntry { + engine: EngineTag::Graph, + database_id: DatabaseId::DEFAULT, + tenant_id: TenantId::new(1), + collection: collection.to_owned(), + key: ReadKey::Predicate, + read_version: nodedb_types::WriteVersion::ZERO, + origin: ReadOrigin::Session, + home: home.map(VShardId::new), + } + } + + fn single_node() -> (std::sync::Arc, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = std::sync::Arc::new( + crate::wal::WalManager::open_for_testing(&dir.path().join("test.wal")).expect("wal"), + ); + let (dispatcher, _sides) = crate::bridge::dispatch::Dispatcher::new(1, 64); + let state = SharedState::new(dispatcher, wal).expect("shared state"); + (state, dir) + } + + #[test] + fn a_single_node_checks_homed_reads_itself() { + let (state, _dir) = single_node(); + let reads = [read("docs", None), read("", Some(3)), read("", Some(9))]; + let split = SplitReads::of(&state, &reads); + assert_eq!(split.enlisted(), &reads[..1]); + assert_eq!(split.checked_here(), &reads[1..]); + } + + #[test] + fn a_read_set_with_no_homed_read_enlists_every_read() { + let (state, _dir) = single_node(); + let reads = [read("docs", None)]; + let split = SplitReads::of(&state, &reads); + assert_eq!(split.enlisted(), &reads[..]); + assert!(split.checked_here().is_empty()); + } +} diff --git a/nodedb/src/control/server/shared/session/commit/read_validation.rs b/nodedb/src/control/server/shared/session/commit/read_validation.rs index 44aa92d87..6397554f7 100644 --- a/nodedb/src/control/server/shared/session/commit/read_validation.rs +++ b/nodedb/src/control/server/shared/session/commit/read_validation.rs @@ -1,29 +1,34 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Commit-time validation of a read-only transaction's homed reads. +//! Commit-time validation of a transaction's reads on their vShards' leaders. //! -//! A homed read is one vShard a cross-shard graph read observed (see -//! `session::graph_reads`). A read-write COMMIT goes through Calvin, whose -//! participants check it. A read-only COMMIT dispatches nothing, so this -//! module checks it: it asks each vShard's leader for the home's current -//! version and compares it to the recorded read version. +//! A read-write COMMIT that goes through Calvin is validated by its +//! participants. Every other COMMIT is validated here: a read-only one, and a +//! single-shard one this node applies itself. Each read entry validates on one +//! vShard: its home, else its collection's vShard. This module asks each +//! vShard's leader for its current version and compares it with the version +//! the read recorded. +//! +//! A version is the data-group log position of the write that set it. Every +//! replica numbers a vShard's writes alike, so a read served by a follower, +//! or by a leader that has since lost leadership, compares with the current +//! leader's versions directly. //! //! The check sends one `MetaOp::HomeVersions` per leader node, carrying every -//! home that node leads. The leader hands each probe to the one core that owns -//! the probe's vShard (`exchange::all_cores::home_versions`), and that core -//! answers: +//! vShard that node leads. The leader hands each probe to the one core that +//! owns the probe's vShard (`exchange::all_cores::home_versions`), and that +//! core answers: //! -//! - A read of one collection gets the collection's write floor on that core. -//! - A read of every collection gets the core watermark. +//! - A read of one collection gets the collection's version on the vShard. +//! - A read of every collection gets the vShard's latest version. //! -//! The leader answers a home only under its leader lease on the home's group, -//! applied through the lease read index. A node that lost leadership refuses -//! the home with the leader and term it knows, and the check asks that leader. +//! The leader answers a probe only under its leader lease on the vShard's +//! group, applied through the lease read index. A node that lost leadership +//! refuses the probe with the leader and term it knows, and the check asks +//! that leader. //! -//! A version above the recorded read version means a write landed on that -//! core after the read. Single-shard SI (`conflict::si_conflict_abort`) -//! compares against this node's WAL, so it never judges a homed read: the -//! read's watermark can come from another node's WAL. +//! A version above the recorded read version means a write landed on the +//! vShard after the read. use std::collections::BTreeMap; @@ -39,7 +44,8 @@ use crate::control::gateway::{RouteDecision, TaskRoute}; use crate::control::server::exchange::all_cores::execute_plan_all_local_cores; use crate::control::server::graph_dispatch::cluster_resolve::gateway_shared; use crate::control::state::SharedState; -use crate::types::{DatabaseId, Lsn, TenantId, TraceId, VShardId}; +use crate::types::{DatabaseId, TenantId, TraceId, VShardId}; +use nodedb_types::WriteVersion; use super::super::connection::SessionId; use super::super::outcome::{AbortReason, CommitOutcome}; @@ -60,14 +66,13 @@ const MAX_CHECK_ROUNDS: u32 = 4; /// The wait before a later round, times the round number. const RETRY_BACKOFF: std::time::Duration = std::time::Duration::from_millis(50); -/// One home read: its tenant scope, the vShard and collection it read, and -/// the node that served it. +/// One validated read: its tenant scope, and the vShard and collection it +/// read. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] struct Home { database_id: u64, tenant_id: u64, probe: HomeVersionProbe, - served_by: u64, } /// One check request: every home one leader leads, in one tenant scope. @@ -78,17 +83,23 @@ struct CheckTarget { tenant_id: u64, } -/// Abort the transaction when a homed read in `read_set` is no longer -/// current, or when a home cannot be checked. Releases the read -/// reservations and rolls the session back before returning the outcome. -/// `None` means every homed read still holds. -pub(super) async fn homed_read_abort( +/// Abort the transaction when a read in `read_set` is no longer current, or +/// when a read cannot be checked. Releases the read reservations and rolls +/// the session back before returning the outcome. `None` means every read +/// still holds. +/// +/// A session read of a collection this transaction writes is skipped. It can +/// have read the transaction's own staged write, which moves no committed +/// version, and the own-write exclusion of the Calvin builders drops it the +/// same way. A plan-derivation read is always checked. +pub(super) async fn stale_read_abort( state: &SharedState, sessions: &SessionStore, session_id: SessionId, read_set: &[ReadSetEntry], + written_collections: &std::collections::HashSet, ) -> Option { - let reason = match homed_reads_changed(state, read_set).await { + let reason = match reads_changed(state, read_set, written_collections).await { Ok(false) => return None, Ok(true) => { super::super::hot_key::record_read_set_aborts(state, read_set); @@ -100,33 +111,32 @@ pub(super) async fn homed_read_abort( Some(CommitOutcome::Aborted { reason }) } -/// Whether any homed read in `read_set` is no longer current. -/// -/// Entries without a home are skipped: SI validates them. +/// Whether any checked read in `read_set` is no longer current. /// /// A read of every collection (a graph read with no collection scope) is -/// checked against the core watermark. That is the conservative rule: any -/// write the core took raises it, to any collection and to any vShard the core -/// owns. Such a check can abort a transaction whose read still holds. It never -/// misses a write that changed the read. +/// checked against the vShard's latest version. Any write to the vShard +/// raises it, so such a check can abort a transaction whose read still holds. +/// It never misses a write that changed the read. /// -/// A leader answers only under its leader lease. A home it refuses is asked +/// A leader answers only under its leader lease. A probe it refuses is asked /// again at the leader the refusal names, or at the routing table's leader -/// when it names none, for at most [`MAX_CHECK_ROUNDS`] rounds. A home still +/// when it names none, for at most [`MAX_CHECK_ROUNDS`] rounds. A probe still /// unanswered then aborts the commit: no answer from a node that lost /// leadership ever lets it through. -/// -/// A version is a position in the answering node's WAL. A home answered by a -/// node other than the one that served the read counts as changed: its -/// versions do not compare with the read's. -async fn homed_reads_changed( +async fn reads_changed( state: &SharedState, read_set: &[ReadSetEntry], + written_collections: &std::collections::HashSet, ) -> crate::Result { - // The earliest read version per home is the one that must still hold. - let mut homes: BTreeMap = BTreeMap::new(); + // The earliest read version per probe is the one that must still hold. + let mut homes: BTreeMap = BTreeMap::new(); for entry in read_set { - let Some(vshard) = entry.home else { + if !entry.origin.survives_own_write_exclusion() + && written_collections.contains(&entry.collection) + { + continue; + } + let Some(vshard) = entry.validation_vshard() else { continue; }; let home = Home { @@ -136,10 +146,9 @@ async fn homed_reads_changed( vshard: vshard.as_u32(), collection: (!entry.collection.is_empty()).then(|| entry.collection.clone()), }, - served_by: entry.home_node, }; - let version = homes.entry(home).or_insert(entry.read_version_lsn); - *version = (*version).min(entry.read_version_lsn); + let version = homes.entry(home).or_insert(entry.read_version); + *version = (*version).min(entry.read_version); } if homes.is_empty() { return Ok(false); @@ -183,10 +192,6 @@ async fn homed_reads_changed( } }); for (target, asked, answers) in futures::future::try_join_all(checks).await? { - let answered_by = match target.leader { - Leader::Local => state.node_id, - Leader::Remote(node_id) => node_id, - }; let by_probe: BTreeMap = answers .into_iter() .map(|answer| (answer.probe, answer.answer)) @@ -195,19 +200,13 @@ async fn homed_reads_changed( let read_version = *homes .get(&home) .ok_or_else(|| unasked_home(target, &home))?; - let answer = *by_probe + let answer = by_probe .get(&home.probe) + .copied() .ok_or_else(|| unasked_home(target, &home))?; match answer { - // Versions are the answering node's WAL positions. Only the - // node that served the read numbers them in the read's own - // domain, so an answer from any other node counts as a - // change. - HomeAnswer::Version(_) if answered_by != home.served_by => { - return Ok(true); - } HomeAnswer::Version(version) => { - if Lsn::new(version) > read_version { + if version > read_version { return Ok(true); } } @@ -236,7 +235,7 @@ async fn homed_reads_changed( return Ok(false); } Err(last_refusal.unwrap_or_else(|| crate::Error::Internal { - detail: "homed read check: a home stayed unanswered; retry the commit".into(), + detail: "read check: a vShard stayed unanswered; retry the commit".into(), })) } @@ -253,7 +252,7 @@ fn leader_of(state: &SharedState, vshard: VShardId) -> crate::Result { }), RouteDecision::Broadcast { .. } => Err(crate::Error::Internal { detail: format!( - "homed read check: vShard {} resolved to a broadcast route", + "read check: vShard {} resolved to a broadcast route", vshard.as_u32() ), }), @@ -310,14 +309,14 @@ async fn current_versions( for payload in payloads { let decoded: Vec = zerompk::from_msgpack(&payload).map_err(|e| crate::Error::Internal { - detail: format!("homed read check: a leader's answer does not decode: {e}"), + detail: format!("read check: a leader's answer does not decode: {e}"), })?; answers.extend(decoded); } if answers.len() != asked { return Err(crate::Error::Internal { detail: format!( - "homed read check: {:?} answered {} of {asked} homes; retry the commit", + "read check: {:?} answered {} of {asked} vShards; retry the commit", target.leader, answers.len() ), @@ -329,7 +328,7 @@ async fn current_versions( fn unasked_home(target: CheckTarget, home: &Home) -> crate::Error { crate::Error::Internal { detail: format!( - "homed read check: {:?} gave no answer for vShard {} ({:?}); retry the commit", + "read check: {:?} gave no answer for vShard {} ({:?}); retry the commit", target.leader, home.probe.vshard, home.probe.collection ), } diff --git a/nodedb/src/control/server/shared/session/commit/run.rs b/nodedb/src/control/server/shared/session/commit/run.rs index 33b4b84f5..b0b861dc2 100644 --- a/nodedb/src/control/server/shared/session/commit/run.rs +++ b/nodedb/src/control/server/shared/session/commit/run.rs @@ -21,9 +21,9 @@ use super::super::outcome::{AbortReason, CommitOutcome, TxnDataPlane}; use super::super::overlay_drop::drop_txn_overlay; use super::super::reservation_release; use super::super::store::SessionStore; -use super::conflict::si_conflict_abort; -use super::homed_reads::homed_read_abort; use super::metering::meter_committed_buffered_writes; +use super::read_split::SplitReads; +use super::read_validation::stale_read_abort; use super::restart_identity::restart_truncated_identities; use super::single_shard::{commit_publishes, dispatch_single_shard}; @@ -83,9 +83,10 @@ pub async fn run_commit( state: &SharedState, dp: &impl TxnDataPlane, ) -> CommitOutcome { - // A graph read made earlier in the same request as this COMMIT is still - // pending in the connection scope. It joins the read-set before validation. - super::super::graph_reads::record_pending(sessions, session_id); + // A cross-shard read made earlier in the same request as this COMMIT is + // still pending in the connection scope. It joins the read-set before + // validation. + super::super::pending_shard_reads::record_pending(sessions, session_id); let read_set = sessions.take_read_set(session_id); // The engine side effects this transaction's index DDL deferred. They run // once its catalog entries landed, below. An abort before then drops them @@ -116,7 +117,10 @@ pub async fn run_commit( // The interactive-COMMIT read-set widens dispatch classification: a txn that // writes shard X but read shard Y participates in {X, Y} and must route // through Calvin with Y as a participant. Autocommit has no session read-set. - let read_vshards = match read_vshards_of(&read_set) { + // A single node checks its homed cross-shard reads itself: they enlist no + // Calvin participant (`read_split`). + let split_reads = SplitReads::of(state, &read_set); + let read_vshards = match read_vshards_of(split_reads.enlisted()) { Ok(vshards) => vshards, Err(error) => { reservation_release::release_and_rollback(state, sessions, session_id).await; @@ -140,19 +144,12 @@ pub async fn run_commit( // anything afterward that can fail in a way compensation will undo. if buffered.is_empty() { // Read-only interactive transaction: no writes to classify, but it can - // still serialization-conflict against concurrent writers. Run the - // single-shard SI validation only — classifying an empty buffer will + // still serialization-conflict against concurrent writers. Each read + // is checked on its vShard's leader. Classifying an empty buffer will // misread a lone cross-shard READ as `MultiShard` and wrongly reject it. if let Some(outcome) = - si_conflict_abort(sessions, session_id, state, &read_set, &written_collections) + stale_read_abort(state, sessions, session_id, &read_set, &written_collections).await { - // Release read reservations (owner still set), then roll back. - reservation_release::release_and_rollback(state, sessions, session_id).await; - return outcome; - } - // SI skips homed reads (the vShards a cross-shard graph read observed). - // Each is checked on its home vShard instead. - if let Some(outcome) = homed_read_abort(state, sessions, session_id, &read_set).await { return outcome; } // No buffered DML for this DDL to race, so finalize it here, @@ -207,11 +204,17 @@ pub async fn run_commit( match classify_dispatch(&buffered, &read_vshards) { DispatchClass::MultiShard { .. } => { // Flush the buffered cross-shard batch through Calvin's durable - // Vote/Verdict barrier (`run_commit_calvin`), leader-routed. SI is - // a single-shard validation and is intentionally NOT run here — - // Calvin performs its own cross-shard OCC over `versioned_reads` - // and returns a serialization abort (SQLSTATE 40001) on an ABORT + // Vote/Verdict barrier (`run_commit_calvin`), leader-routed. + // Calvin's participants validate `versioned_reads` themselves + // and return a serialization abort (SQLSTATE 40001) on an ABORT // verdict. + if let Some(outcome) = split_reads + .stale_here(state, sessions, session_id, &written_collections) + .await + { + compensate_finalized_ddl(state, &ddl_compensation).await; + return outcome; + } if let Some(reason) = commit_calvin::run_commit_calvin( sessions, session_id, @@ -219,7 +222,7 @@ pub async fn run_commit( commit_calvin::CalvinCommit { buffered: &buffered, tenant_id, - reads: &read_set, + reads: split_reads.enlisted(), event_source: dp.event_source(), publishes: &publishes, applied_key: dp.applied_key().zip(dp.applied_key_vshard()), @@ -250,6 +253,13 @@ pub async fn run_commit( // A commit that writes an edge takes it too: every edge // version takes a Calvin ordinal, so it orders against a // TRUNCATE's cut on every replica. + if let Some(outcome) = split_reads + .stale_here(state, sessions, session_id, &written_collections) + .await + { + compensate_finalized_ddl(state, &ddl_compensation).await; + return outcome; + } if let Some(reason) = commit_calvin::run_commit_calvin( sessions, session_id, @@ -257,7 +267,7 @@ pub async fn run_commit( commit_calvin::CalvinCommit { buffered: &buffered, tenant_id, - reads: &read_set, + reads: split_reads.enlisted(), event_source: dp.event_source(), publishes: &publishes, applied_key: dp.applied_key().zip(dp.applied_key_vshard()), @@ -272,20 +282,14 @@ pub async fn run_commit( return CommitOutcome::Aborted { reason }; } } else { - if let Some(outcome) = si_conflict_abort( + if let Some(outcome) = stale_read_abort( + state, sessions, session_id, - state, &read_set, &written_collections, - ) { - compensate_finalized_ddl(state, &ddl_compensation).await; - reservation_release::release_and_rollback(state, sessions, session_id) - .await; - return outcome; - } - if let Some(outcome) = - homed_read_abort(state, sessions, session_id, &read_set).await + ) + .await { compensate_finalized_ddl(state, &ddl_compensation).await; return outcome; diff --git a/nodedb/src/control/server/shared/session/commit_calvin.rs b/nodedb/src/control/server/shared/session/commit_calvin.rs index c6a191101..592e41b5b 100644 --- a/nodedb/src/control/server/shared/session/commit_calvin.rs +++ b/nodedb/src/control/server/shared/session/commit_calvin.rs @@ -197,10 +197,9 @@ async fn submit_commit_calvin( } else { event_source }; - // Empty read-set: best-effort performs no cross-shard OCC (the - // multi-shard COMMIT path never ran `si_conflict_abort`), so each - // group carries no versioned reads — matching the single-vShard - // submit `route_write_to_calvin` uses. + // Empty read-set: best-effort performs no cross-shard OCC, so + // each group carries no versioned reads — matching the + // single-vShard submit `route_write_to_calvin` uses. // A timeseries ingest resolves to its rows here, before it is // sequenced: every replica resolves a sequenced group on its own. let tasks = diff --git a/nodedb/src/control/server/shared/session/conn_scope.rs b/nodedb/src/control/server/shared/session/conn_scope.rs index cd4994f79..35de2f8d2 100644 --- a/nodedb/src/control/server/shared/session/conn_scope.rs +++ b/nodedb/src/control/server/shared/session/conn_scope.rs @@ -15,7 +15,7 @@ use std::time::Instant; use super::audit_context::AuditCtx; use super::ddl_buffer::DdlBuffer; use super::ephemeral_sequence::EphemeralSequences; -use super::graph_reads::GraphShardReads; +use super::pending_shard_reads::ShardReads; /// Per-connection session slots. Each field is owned by its own module, which /// exposes the accessors; nothing outside reaches through this struct. @@ -33,12 +33,9 @@ pub(super) struct ConnScope { /// Client-facing notices raised while the current statement ran, below /// the response shaper (see [`super::statement_notice`]). pub(super) statement_notices: RefCell>, - /// Cross-shard graph reads the current request made, not yet recorded - /// into the transaction read-set (see [`super::graph_reads`]). - pub(super) graph_reads: RefCell>, - /// The node that served each vShard the current statement read (see - /// [`super::served_reads`]). - pub(super) served_reads: RefCell>, + /// Cross-shard reads the current request made, not yet recorded into the + /// transaction read-set (see [`super::pending_shard_reads`]). + pub(super) pending_shard_reads: RefCell>, } impl ConnScope { @@ -49,8 +46,7 @@ impl ConnScope { ephemeral_sequences: RefCell::new(EphemeralSequences::new()), statement_deadline: Cell::new(None), statement_notices: RefCell::new(Vec::new()), - graph_reads: RefCell::new(Vec::new()), - served_reads: super::served_reads::empty(), + pending_shard_reads: RefCell::new(Vec::new()), } } } diff --git a/nodedb/src/control/server/shared/session/deadline.rs b/nodedb/src/control/server/shared/session/deadline.rs index 6831efd36..cbe6447db 100644 --- a/nodedb/src/control/server/shared/session/deadline.rs +++ b/nodedb/src/control/server/shared/session/deadline.rs @@ -57,8 +57,7 @@ impl Drop for StatementScope { } } -/// Pin the deadline for the statement about to run, and start the -/// statement's served-read notes empty. +/// Pin the deadline for the statement about to run. /// /// The instant is fixed ONCE here, so every request the statement fans out into /// shares it and the statement is bounded end to end rather than per hop. @@ -67,8 +66,6 @@ impl Drop for StatementScope { /// store is a no-op and every envelope site falls back to the node default, so /// installing the guard is always safe. pub fn enter(statement_timeout: Option, default_deadline_secs: u64) -> StatementScope { - // A statement starts with no reads served (`served_reads`). - super::served_reads::clear(); let previous = current(); store(Some( Instant::now() + statement_budget(statement_timeout, default_deadline_secs), diff --git a/nodedb/src/control/server/shared/session/graph_reads.rs b/nodedb/src/control/server/shared/session/graph_reads.rs deleted file mode 100644 index 545cd9291..000000000 --- a/nodedb/src/control/server/shared/session/graph_reads.rs +++ /dev/null @@ -1,197 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Cross-shard graph reads recorded into the transaction read-set. -//! -//! A cluster graph read (a MATCH, a walk, a gathered algorithm) reads edges on -//! many key vShards through a coordinator that holds no session. The -//! coordinator notes every vShard it read, each at the watermark that shard -//! served, in the connection scope ([`note`]). Each protocol records the noted -//! reads when the request ends ([`record_pending`]). -//! -//! Each vShard becomes one [`ReadSetEntry`] homed on that vShard. Commit -//! validation then checks the entry on the vShard that holds the edges: -//! -//! - A Calvin participant checks the collection's write floor on that vShard. -//! - A read-only commit fetches that floor from the vShard's leader -//! (`commit::homed_reads`). -//! -//! The read version is the served watermark. A collection's write floor on a -//! core never exceeds the core watermark, and a later write to the collection -//! on that core carries a higher LSN. So a floor above the recorded watermark -//! means the collection changed on that vShard after the read. - -use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; - -use super::conn_scope::with_scope; -use super::connection::SessionId; -use super::read_set::{EngineTag, ReadKey, ReadOrigin, ReadSetEntry}; -use super::store::SessionStore; - -/// One vShard a graph read observed, at the watermark its core served, and -/// the node that served it (`0` when more than one did). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ShardObservation { - pub vshard: VShardId, - pub watermark: Lsn, - pub node: u64, -} - -/// The vShards one cross-shard graph read observed. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct GraphShardReads { - pub tenant_id: TenantId, - pub database_id: DatabaseId, - /// The database-qualified collection the read scoped, or `None` when it - /// walked every collection. - pub collection: Option, - pub shards: Vec, -} - -/// Note a cross-shard graph read for the request now running. Outside a -/// connection scope there is no session to record it for, so it is dropped. -pub fn note(reads: GraphShardReads) { - if reads.shards.is_empty() { - return; - } - let mut pending = Some(reads); - with_scope((), |scope| { - if let Some(reads) = pending.take() { - scope.graph_reads.borrow_mut().push(reads); - } - }); -} - -/// Take every graph read the request noted, oldest first. -pub fn take() -> Vec { - with_scope(Vec::new(), |scope| { - std::mem::take(&mut *scope.graph_reads.borrow_mut()) - }) -} - -/// Record the graph reads the request noted into `session_id`'s transaction -/// read-set. Outside a transaction block the reads are dropped. -pub fn record_pending(sessions: &SessionStore, session_id: SessionId) { - let pending = take(); - if pending.is_empty() || !sessions.is_in_transaction_block(session_id) { - return; - } - for reads in pending { - let entries = homed_entries(sessions, session_id, reads); - sessions.record_read_entries(session_id, entries); - } -} - -/// One predicate entry per observed vShard, homed there. -fn homed_entries( - sessions: &SessionStore, - session_id: SessionId, - reads: GraphShardReads, -) -> Vec { - let GraphShardReads { - tenant_id, - database_id, - collection, - shards, - } = reads; - let collection = collection.unwrap_or_default(); - // The session's own committed writes to the collection raise the read - // version, as `record_read_set` does, so a read never aborts on the - // session's own earlier write. - let own_write_version = if collection.is_empty() { - Lsn::ZERO - } else { - sessions.own_write_version(session_id, database_id, tenant_id, &collection) - }; - shards - .into_iter() - .map(|shard| ReadSetEntry { - engine: EngineTag::Graph, - database_id, - tenant_id, - collection: collection.clone(), - key: ReadKey::Predicate, - read_lsn: shard.watermark, - read_version_lsn: shard.watermark.max(own_write_version), - origin: ReadOrigin::Session, - home: Some(shard.vshard), - home_node: shard.node, - }) - .collect() -} - -#[cfg(test)] -mod tests { - use super::super::conn_scope; - use super::*; - - fn session_id() -> SessionId { - SessionId::from( - "127.0.0.1:5611" - .parse::() - .expect("test address"), - ) - } - - fn reads(collection: Option<&str>) -> GraphShardReads { - GraphShardReads { - tenant_id: TenantId::new(1), - database_id: DatabaseId::DEFAULT, - collection: collection.map(str::to_owned), - shards: vec![ - ShardObservation { - vshard: VShardId::new(3), - watermark: Lsn::new(10), - node: 1, - }, - ShardObservation { - vshard: VShardId::new(9), - watermark: Lsn::new(20), - node: 2, - }, - ], - } - } - - fn store_in_block() -> (SessionStore, SessionId) { - let sessions = SessionStore::new(); - let session_id = session_id(); - sessions.ensure_session(match session_id { - SessionId::LegacySocket(addr) => addr, - SessionId::Connection(_) => unreachable!("legacy test identity"), - }); - sessions.begin(session_id, Lsn::new(5), 0).expect("begin"); - (sessions, session_id) - } - - #[tokio::test] - async fn each_observed_vshard_becomes_one_homed_entry() { - let (sessions, session_id) = store_in_block(); - conn_scope::scoped(async { - note(reads(Some("db1.edges"))); - record_pending(&sessions, session_id); - }) - .await; - let entries = sessions.take_read_set(session_id); - assert_eq!(entries.len(), 2); - assert_eq!(entries[0].home, Some(VShardId::new(3))); - assert_eq!(entries[0].read_version_lsn, Lsn::new(10)); - assert_eq!(entries[1].home, Some(VShardId::new(9))); - assert_eq!(entries[1].read_version_lsn, Lsn::new(20)); - assert_eq!(entries[1].home_node, 2); - assert!(entries.iter().all(|e| e.key == ReadKey::Predicate)); - assert!(entries.iter().all(|e| e.collection == "db1.edges")); - } - - #[tokio::test] - async fn a_read_outside_a_transaction_block_records_nothing() { - let sessions = SessionStore::new(); - let session_id = session_id(); - conn_scope::scoped(async { - note(reads(None)); - record_pending(&sessions, session_id); - assert!(take().is_empty(), "the pending reads are drained"); - }) - .await; - assert!(sessions.take_read_set(session_id).is_empty()); - } -} diff --git a/nodedb/src/control/server/shared/session/leader_forward.rs b/nodedb/src/control/server/shared/session/leader_forward.rs index 497ef245f..843a3c180 100644 --- a/nodedb/src/control/server/shared/session/leader_forward.rs +++ b/nodedb/src/control/server/shared/session/leader_forward.rs @@ -120,7 +120,7 @@ pub(crate) async fn forward_to_leader( let payload = outcome.payloads.into_iter().next().unwrap_or_default(); let mut resp = bare_ok_response(RequestId::new(0)); resp.payload = Payload::from_vec(payload); - resp.read_version_lsn = outcome.read_version_lsn; + resp.read_versions = outcome.read_versions; Ok(resp) } diff --git a/nodedb/src/control/server/shared/session/lifecycle.rs b/nodedb/src/control/server/shared/session/lifecycle.rs index 4094daebb..620b81fb9 100644 --- a/nodedb/src/control/server/shared/session/lifecycle.rs +++ b/nodedb/src/control/server/shared/session/lifecycle.rs @@ -24,10 +24,6 @@ pub fn run_begin( session_id: SessionId, state: &SharedState, ) -> Result<(), crate::Error> { - let snapshot_lsn = { - let next = state.wal.next_lsn(); - crate::types::Lsn::new(next.as_u64().saturating_sub(1)) - }; // Last globally-applied Calvin epoch as the cross-shard snapshot anchor. // 0 in single-node / no-Calvin deployments (the atomic is never advanced). let snapshot_epoch = state @@ -36,7 +32,7 @@ pub fn run_begin( .load(std::sync::atomic::Ordering::Acquire); ddl_buffer::activate(); sessions - .begin(session_id, snapshot_lsn, snapshot_epoch) + .begin(session_id, snapshot_epoch) .map_err(|msg| crate::Error::BadRequest { detail: msg.to_owned(), }) @@ -199,7 +195,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }) }) diff --git a/nodedb/src/control/server/shared/session/mod.rs b/nodedb/src/control/server/shared/session/mod.rs index 508e095f4..45b8465a4 100644 --- a/nodedb/src/control/server/shared/session/mod.rs +++ b/nodedb/src/control/server/shared/session/mod.rs @@ -19,7 +19,6 @@ pub mod ddl_rollback; pub mod deadline; pub mod ephemeral_sequence; pub mod expander_stage; -pub mod graph_reads; mod hot_key; mod leader_forward; pub mod lifecycle; @@ -31,13 +30,13 @@ pub mod outcome; pub mod overlay_drop; mod own_writes; mod params; +pub mod pending_shard_reads; pub mod read_consistency; pub mod read_set; pub mod record_reads; mod reservation_release; pub mod savepoint_ops; mod sequence_values; -pub mod served_reads; pub mod set_validation; pub mod staging_gate; mod state; diff --git a/nodedb/src/control/server/shared/session/own_writes.rs b/nodedb/src/control/server/shared/session/own_writes.rs index a88ce822a..01564fe6e 100644 --- a/nodedb/src/control/server/shared/session/own_writes.rs +++ b/nodedb/src/control/server/shared/session/own_writes.rs @@ -2,73 +2,111 @@ //! Per-session read-your-writes floor tracking on `SessionStore`. //! -//! Records the highest committed write-version this session has observed for -//! each `(database, tenant, collection)` it has written, so a later -//! transaction's read-set capture can be floored at the session's OWN prior -//! committed writes. This extends the read-your-own-write exclusion already -//! applied to a transaction's buffered writes (see the Calvin static builder) -//! to the session's PRIOR committed autocommit writes: without it, a -//! cross-shard OCC validation would see the session's own committed -//! `coll_write_lsn` exceed a read that captured a stale floor and false-abort -//! with a serialization failure. +//! Records the highest committed write version this session holds for each +//! `(database, tenant, collection, vShard)` it has written. A later +//! transaction's read-set capture is floored at it. This extends the +//! read-your-own-write exclusion the Calvin static builder applies to buffered +//! writes to the session's prior committed writes. Without it, OCC validation +//! sees the session's own committed write above a read that captured an older +//! version, and aborts the transaction on its own write. //! -//! Soundness: the floor is only ever RAISED by this session's own committed -//! writes to that exact `(database, tenant, collection)`. A concurrent -//! OTHER-session write yields a higher `coll_write_lsn` that still exceeds the -//! floor, so a genuine conflict still aborts — this only removes the self-abort, -//! never a real one. +//! Only this session's own committed writes raise the floor. A concurrent +//! write by another session has a higher version that still exceeds the +//! floor, so a real conflict still aborts. + +use nodedb_types::WriteVersion; use super::connection::SessionId; -use crate::types::{DatabaseId, Lsn, TenantId}; +use crate::bridge::envelope::{PhysicalPlan, Response, Status}; +use crate::control::security::identity::{Permission, required_permission}; +use crate::control::server::shared::plan_util::extract_collection; +use crate::types::{DatabaseId, ReadVersions, TenantId, VShardId}; use super::store::SessionStore; +/// The scope of one own-write floor: database, tenant, collection, and the +/// vShard whose versions the floor is a position in. +pub type OwnWriteKey = (DatabaseId, TenantId, String, VShardId); + impl SessionStore { - /// Record a committed write-version for `(database, tenant, collection)` on - /// this session, keeping the maximum seen. `version` is the write's - /// committed per-collection version (`coll_write_lsn`), sourced from the - /// replicated-write response. A `Lsn::ZERO` version carries no floor and is - /// ignored. Persists for the life of the session. + /// Record the versions a dispatched write of `plan` stamped, taken from + /// its `response`. Every protocol calls this once per dispatched task. + /// A read, a failed write, or a write of no single collection records + /// nothing. + pub fn note_own_write_response( + &self, + addr: impl Into, + database_id: DatabaseId, + tenant_id: TenantId, + plan: &PhysicalPlan, + response: &Response, + ) { + if response.status != Status::Ok + || response.read_versions.is_empty() + || !matches!(required_permission(plan), Permission::Write) + { + return; + } + let Some(collection) = extract_collection(plan) else { + return; + }; + self.note_own_write( + addr, + database_id, + tenant_id, + collection, + &response.read_versions, + ); + } + + /// Record the versions a committed write to `(database, tenant, + /// collection)` stamped, keeping the maximum per vShard. `versions` come + /// from the write's response. A write that stamped none sets no floor. + /// The floors persist for the life of the session. pub fn note_own_write( &self, addr: impl Into, database_id: DatabaseId, tenant_id: TenantId, collection: &str, - version: Lsn, + versions: &ReadVersions, ) { - if version == Lsn::ZERO { + if versions.is_empty() { return; } self.write_session(addr, |session| { - let slot = session - .own_write_versions - .entry((database_id, tenant_id, collection.to_string())) - .or_insert(Lsn::ZERO); - if version > *slot { - *slot = version; + for shard in versions.iter() { + let key = ( + database_id, + tenant_id, + collection.to_string(), + VShardId::new(shard.vshard), + ); + let slot = session.own_write_versions.entry(key).or_default(); + *slot = (*slot).max(shard.version); } }); } - /// Return the session's own highest committed write-version for - /// `(database, tenant, collection)`, or `Lsn::ZERO` if the session never - /// wrote that collection (no floor to apply). + /// The session's own highest committed write version for `(database, + /// tenant, collection)` on `vshard`, or `WriteVersion::ZERO` when the + /// session never wrote it there. pub fn own_write_version( &self, addr: impl Into, database_id: DatabaseId, tenant_id: TenantId, collection: &str, - ) -> Lsn { + vshard: VShardId, + ) -> WriteVersion { self.read_session(addr, |session| { session .own_write_versions - .get(&(database_id, tenant_id, collection.to_string())) + .get(&(database_id, tenant_id, collection.to_string(), vshard)) .copied() - .unwrap_or(Lsn::ZERO) + .unwrap_or_default() }) - .unwrap_or(Lsn::ZERO) + .unwrap_or_default() } } @@ -78,6 +116,8 @@ mod tests { use super::*; + const HOME: VShardId = VShardId::new(4); + fn addr() -> SocketAddr { "127.0.0.1:5601".parse().expect("test addr") } @@ -89,83 +129,141 @@ mod tests { (sessions, a) } + fn at(index: u64) -> ReadVersions { + ReadVersions::single(HOME, WriteVersion::logged(1, index)) + } + + fn floor( + sessions: &SessionStore, + a: SocketAddr, + tenant: u64, + collection: &str, + ) -> WriteVersion { + sessions.own_write_version( + a, + DatabaseId::DEFAULT, + TenantId::new(tenant), + collection, + HOME, + ) + } + #[test] fn absent_collection_returns_zero() { let (sessions, a) = store_with_session(); - assert_eq!( - sessions.own_write_version(a, DatabaseId::DEFAULT, TenantId::new(1), "bread"), - Lsn::ZERO - ); + assert_eq!(floor(&sessions, a, 1, "bread"), WriteVersion::ZERO); } #[test] fn records_and_returns_own_write_version() { let (sessions, a) = store_with_session(); - sessions.note_own_write( - a, - DatabaseId::DEFAULT, - TenantId::new(1), - "bread", - Lsn::new(2), - ); - assert_eq!( - sessions.own_write_version(a, DatabaseId::DEFAULT, TenantId::new(1), "bread"), - Lsn::new(2) - ); + sessions.note_own_write(a, DatabaseId::DEFAULT, TenantId::new(1), "bread", &at(2)); + assert_eq!(floor(&sessions, a, 1, "bread"), WriteVersion::logged(1, 2)); } #[test] fn keeps_the_maximum_version() { let (sessions, a) = store_with_session(); - sessions.note_own_write( - a, - DatabaseId::DEFAULT, - TenantId::new(1), - "bread", - Lsn::new(5), - ); + sessions.note_own_write(a, DatabaseId::DEFAULT, TenantId::new(1), "bread", &at(5)); // A lower version never lowers the floor. + sessions.note_own_write(a, DatabaseId::DEFAULT, TenantId::new(1), "bread", &at(3)); + assert_eq!(floor(&sessions, a, 1, "bread"), WriteVersion::logged(1, 5)); + } + + #[test] + fn a_write_with_no_versions_sets_no_floor() { + let (sessions, a) = store_with_session(); sessions.note_own_write( a, DatabaseId::DEFAULT, TenantId::new(1), "bread", - Lsn::new(3), + &ReadVersions::new(), ); - assert_eq!( - sessions.own_write_version(a, DatabaseId::DEFAULT, TenantId::new(1), "bread"), - Lsn::new(5) + assert_eq!(floor(&sessions, a, 1, "bread"), WriteVersion::ZERO); + } + + fn kv_plan(collection: &str, write: bool) -> PhysicalPlan { + let collection = nodedb_types::QualifiedCollection::new(DatabaseId::DEFAULT, collection); + PhysicalPlan::Kv(if write { + nodedb_physical::physical_plan::KvOp::Put { + collection, + key: b"k".to_vec(), + value: b"v".to_vec(), + ttl_ms: 0, + surrogate: nodedb_types::Surrogate::new(1), + returning: None, + rls_filters: Vec::new(), + provenance: None, + } + } else { + nodedb_physical::physical_plan::KvOp::Get { + collection, + key: b"k".to_vec(), + rls_filters: Vec::new(), + surrogate_ceiling: None, + } + }) + } + + fn response(status: Status, versions: ReadVersions) -> Response { + let mut response = crate::control::server::shared::write_admission::bare_ok_response( + crate::types::RequestId::new(1), ); + response.status = status; + response.read_versions = versions; + response } #[test] - fn zero_version_is_ignored() { + fn a_dispatched_write_response_sets_the_floor() { let (sessions, a) = store_with_session(); - sessions.note_own_write(a, DatabaseId::DEFAULT, TenantId::new(1), "bread", Lsn::ZERO); - assert_eq!( - sessions.own_write_version(a, DatabaseId::DEFAULT, TenantId::new(1), "bread"), - Lsn::ZERO + sessions.note_own_write_response( + a, + DatabaseId::DEFAULT, + TenantId::new(1), + &kv_plan("bread", true), + &response(Status::Ok, at(9)), ); + assert_eq!(floor(&sessions, a, 1, "bread"), WriteVersion::logged(1, 9)); } #[test] - fn scoped_by_database_tenant_and_collection() { + fn a_read_or_a_failed_write_sets_no_floor() { let (sessions, a) = store_with_session(); - sessions.note_own_write( + sessions.note_own_write_response( a, DatabaseId::DEFAULT, TenantId::new(1), - "bread", - Lsn::new(7), + &kv_plan("bread", false), + &response(Status::Ok, at(9)), ); - // A different collection, tenant, or database sees no floor. - assert_eq!( - sessions.own_write_version(a, DatabaseId::DEFAULT, TenantId::new(1), "milk"), - Lsn::ZERO + sessions.note_own_write_response( + a, + DatabaseId::DEFAULT, + TenantId::new(1), + &kv_plan("bread", true), + &response(Status::Error, at(9)), ); + assert_eq!(floor(&sessions, a, 1, "bread"), WriteVersion::ZERO); + } + + #[test] + fn scoped_by_database_tenant_collection_and_vshard() { + let (sessions, a) = store_with_session(); + sessions.note_own_write(a, DatabaseId::DEFAULT, TenantId::new(1), "bread", &at(7)); + // A different collection, tenant, or vShard sees no floor. + assert_eq!(floor(&sessions, a, 1, "milk"), WriteVersion::ZERO); + assert_eq!(floor(&sessions, a, 2, "bread"), WriteVersion::ZERO); assert_eq!( - sessions.own_write_version(a, DatabaseId::DEFAULT, TenantId::new(2), "bread"), - Lsn::ZERO + sessions.own_write_version( + a, + DatabaseId::DEFAULT, + TenantId::new(1), + "bread", + VShardId::new(5), + ), + WriteVersion::ZERO ); } } diff --git a/nodedb/src/control/server/shared/session/pending_shard_reads.rs b/nodedb/src/control/server/shared/session/pending_shard_reads.rs new file mode 100644 index 000000000..10a29c55f --- /dev/null +++ b/nodedb/src/control/server/shared/session/pending_shard_reads.rs @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Cross-shard reads recorded into the transaction read-set. +//! +//! Some reads span many vShards through a coordinator that holds no session: +//! +//! - A cluster graph read (a MATCH, a walk, a gathered algorithm) reads edges +//! on many key vShards. +//! - A cluster array read (a slice or an aggregate) reads cells on many tile +//! vShards. +//! +//! The coordinator notes every vShard it read, each at the version the serving +//! core reported, in the connection scope ([`note`]). Each protocol records +//! the noted reads when the request ends ([`record_pending`]). +//! +//! Each vShard becomes one [`ReadSetEntry`] homed on that vShard. Commit +//! validation then checks the entry on the vShard that holds the data: +//! +//! - A Calvin participant checks the collection's version on that vShard. +//! - Any other commit fetches that version from the vShard's leader +//! (`commit::read_validation`). +//! +//! A version is a data-group log position, the same on every replica. So a +//! version above the recorded one means the collection changed on that vShard +//! after the read, whichever replica served it. + +use nodedb_types::WriteVersion; + +use crate::types::{DatabaseId, TenantId, VShardId}; + +use super::conn_scope::with_scope; +use super::connection::SessionId; +use super::read_set::{EngineTag, ReadKey, ReadOrigin, ReadSetEntry}; +use super::store::SessionStore; + +/// One vShard a cross-shard read observed, at the version its serving core +/// reported (`WriteVersion::ZERO` when the core reported none). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ShardObservation { + pub vshard: VShardId, + pub version: WriteVersion, +} + +/// The vShards one cross-shard read observed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ShardReads { + /// The engine that served the read: `Graph` or `ClusterArray`. + pub engine: EngineTag, + pub tenant_id: TenantId, + pub database_id: DatabaseId, + /// The database-qualified collection the read scoped, or `None` when it + /// read every collection. + pub collection: Option, + pub shards: Vec, +} + +/// Note a cross-shard read for the request now running. Outside a +/// connection scope there is no session to record it for, so it is dropped. +pub fn note(reads: ShardReads) { + if reads.shards.is_empty() { + return; + } + let mut pending = Some(reads); + with_scope((), |scope| { + if let Some(reads) = pending.take() { + scope.pending_shard_reads.borrow_mut().push(reads); + } + }); +} + +/// Take every cross-shard read the request noted, oldest first. +pub fn take() -> Vec { + with_scope(Vec::new(), |scope| { + std::mem::take(&mut *scope.pending_shard_reads.borrow_mut()) + }) +} + +/// Record the cross-shard reads the request noted into `session_id`'s +/// transaction read-set. Outside a transaction block the reads are dropped. +pub fn record_pending(sessions: &SessionStore, session_id: SessionId) { + let pending = take(); + if pending.is_empty() || !sessions.is_in_transaction_block(session_id) { + return; + } + for reads in pending { + let entries = homed_entries(sessions, session_id, reads); + sessions.record_read_entries(session_id, entries); + } +} + +/// One predicate entry per observed vShard, homed there. +fn homed_entries( + sessions: &SessionStore, + session_id: SessionId, + reads: ShardReads, +) -> Vec { + let ShardReads { + engine, + tenant_id, + database_id, + collection, + shards, + } = reads; + let collection = collection.unwrap_or_default(); + shards + .into_iter() + .map(|shard| { + // The session's own committed writes to the collection raise the + // read version, as `record_read_set` does, so a read never aborts + // on the session's own earlier write. + let own = if collection.is_empty() { + WriteVersion::ZERO + } else { + sessions.own_write_version( + session_id, + database_id, + tenant_id, + &collection, + shard.vshard, + ) + }; + ReadSetEntry { + engine, + database_id, + tenant_id, + collection: collection.clone(), + key: ReadKey::Predicate, + read_version: shard.version.max(own), + origin: ReadOrigin::Session, + home: Some(shard.vshard), + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::super::conn_scope; + use super::*; + + fn session_id() -> SessionId { + SessionId::from( + "127.0.0.1:5611" + .parse::() + .expect("test address"), + ) + } + + fn reads(engine: EngineTag, collection: Option<&str>) -> ShardReads { + ShardReads { + engine, + tenant_id: TenantId::new(1), + database_id: DatabaseId::DEFAULT, + collection: collection.map(str::to_owned), + shards: vec![ + ShardObservation { + vshard: VShardId::new(3), + version: WriteVersion::logged(1, 10), + }, + ShardObservation { + vshard: VShardId::new(9), + version: WriteVersion::logged(2, 20), + }, + ], + } + } + + fn store_in_block() -> (SessionStore, SessionId) { + let sessions = SessionStore::new(); + let session_id = session_id(); + sessions.ensure_session(match session_id { + SessionId::LegacySocket(addr) => addr, + SessionId::Connection(_) => unreachable!("legacy test identity"), + }); + sessions.begin(session_id, 0).expect("begin"); + (sessions, session_id) + } + + #[tokio::test] + async fn each_observed_vshard_becomes_one_homed_entry() { + let (sessions, session_id) = store_in_block(); + conn_scope::scoped(async { + note(reads(EngineTag::Graph, Some("db1.edges"))); + record_pending(&sessions, session_id); + }) + .await; + let entries = sessions.take_read_set(session_id); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].home, Some(VShardId::new(3))); + assert_eq!(entries[0].read_version, WriteVersion::logged(1, 10)); + assert_eq!(entries[1].home, Some(VShardId::new(9))); + assert_eq!(entries[1].read_version, WriteVersion::logged(2, 20)); + assert!(entries.iter().all(|e| e.key == ReadKey::Predicate)); + assert!(entries.iter().all(|e| e.collection == "db1.edges")); + assert!(entries.iter().all(|e| e.engine == EngineTag::Graph)); + } + + #[tokio::test] + async fn entries_carry_the_engine_that_served_the_read() { + let (sessions, session_id) = store_in_block(); + conn_scope::scoped(async { + note(reads(EngineTag::Graph, Some("db1.edges"))); + note(reads(EngineTag::ClusterArray, Some("db1.grid"))); + record_pending(&sessions, session_id); + }) + .await; + let entries = sessions.take_read_set(session_id); + assert_eq!(entries.len(), 4); + assert!( + entries[..2] + .iter() + .all(|e| e.engine == EngineTag::Graph && e.collection == "db1.edges") + ); + assert!( + entries[2..] + .iter() + .all(|e| e.engine == EngineTag::ClusterArray && e.collection == "db1.grid") + ); + } + + #[tokio::test] + async fn a_read_outside_a_transaction_block_records_nothing() { + let sessions = SessionStore::new(); + let session_id = session_id(); + conn_scope::scoped(async { + note(reads(EngineTag::ClusterArray, None)); + record_pending(&sessions, session_id); + assert!(take().is_empty(), "the pending reads are drained"); + }) + .await; + assert!(sessions.take_read_set(session_id).is_empty()); + } +} diff --git a/nodedb/src/control/server/shared/session/read_set.rs b/nodedb/src/control/server/shared/session/read_set.rs index 01b0e4093..69d06f88c 100644 --- a/nodedb/src/control/server/shared/session/read_set.rs +++ b/nodedb/src/control/server/shared/session/read_set.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Protocol-neutral, LSN-versioned transaction read-set capture. +//! Protocol-neutral, versioned transaction read-set capture. //! //! Every read a transaction performs is recorded here as one or more //! [`ReadSetEntry`]s, keyed by the same `(database_id, tenant_id, collection, @@ -15,11 +15,15 @@ //! one no point key will name), while an absent KV point read keeps its precise `Point` //! key (the byte key any future write reuses). A //! scan / search / aggregate records [`ReadKey::Predicate`] (collection scope -//! — the day-one phantom-safe floor). A multi-shard read records one entry per -//! participating shard, each stamped with that shard's own watermark LSN. +//! — the day-one phantom-safe floor). //! Absent-key / empty-result reads are recorded too: a "not found" is a //! validatable phantom observation, not a no-op. //! +//! Each entry carries the write version its read observed on its validation +//! vShard. A version is the data-group log position of the write that set it, +//! so every replica of the vShard reports the same version for the same state. +//! A read served by a follower validates against the leader's versions. +//! //! No validation happens here — the entries are captured for the commit-time //! optimistic-concurrency check to consume. @@ -32,7 +36,8 @@ use crate::control::cluster::calvin::scheduler::lock::LockKey; use crate::control::planner::calvin::reservation::submit_reserve_read; use crate::control::server::shared::plan_util::{extract_collection, plan_engine, read_key_of}; use crate::control::state::SharedState; -use crate::types::{DatabaseId, KeyRepr, Lsn, TenantId, VShardId}; +use crate::types::{DatabaseId, KeyRepr, ReadVersions, TenantId, VShardId}; +use nodedb_types::WriteVersion; use super::connection::SessionId; use super::store::SessionStore; @@ -116,7 +121,7 @@ impl ReadOrigin { } } -/// One LSN-versioned, predicate-aware read-set entry. Scoped by +/// One versioned, predicate-aware read-set entry. Scoped by /// `(database_id, tenant_id)` exactly like the write path so two tenants (or /// databases) never alias. #[derive(Debug, Clone, PartialEq, Eq)] @@ -126,78 +131,66 @@ pub struct ReadSetEntry { pub tenant_id: TenantId, pub collection: String, pub key: ReadKey, - pub read_lsn: Lsn, - /// Per-collection read-version LSN (the read collection's `coll_write_lsn` - /// at read time, a WAL LSN) — the SOUND comparand for cross-shard OCC - /// validation, which compares it against the same collection's recorded - /// write versions in that one domain. `read_lsn` above stays the core-global - /// watermark used by single-shard SI (`si_conflict_abort`). - pub read_version_lsn: Lsn, + /// The version the read observed on its validation vShard. Commit + /// validation compares it with that vShard's current version of the + /// collection. Every replica numbers versions alike, so the read can have + /// been served by any replica. + pub read_version: WriteVersion, /// Whether this observation is the transaction's own session read or a /// plan-time derivation read. Required at construction — the own-write /// exclusion in the `TxClass` builders reads it, and an entry that guessed /// will be silently dropped from validation. pub origin: ReadOrigin, /// The vShard whose write versions validate this entry, or `None` for the - /// collection's own vShard. A cross-shard graph read records one entry per - /// key vShard it read, each homed there. A homed entry with an empty + /// collection's own vShard. A cross-shard graph or array read records one + /// entry per vShard it read, each homed there. A homed entry with an empty /// `collection` observed every collection on its vShard. pub home: Option, - /// The node that served the read: of `home` for a homed read, of the - /// collection's vShard otherwise. Its WAL numbers the read's versions, so - /// only that node's current versions compare with them. `0` when no one - /// node is known to have served it, which every commit check treats as a - /// change. - pub home_node: u64, } -/// The node that served the running statement's read of `vshard`. -/// -/// The read dispatch notes it (`served_reads`). A read no dispatch noted ran -/// on this node's cores: it is this node when there is no cluster or this node -/// leads the vShard now, and unknown (`0`) otherwise. -pub(crate) fn serving_node(state: &SharedState, vshard: u32) -> u64 { - if let Some(node) = super::served_reads::served_by(vshard) { - return node; - } - if state.cluster_routing.is_none() { - return state.node_id; - } - match crate::control::gateway::live_leaders::resolve_live_decision(state, vshard) { - crate::control::gateway::RouteDecision::Local => state.node_id, - _ => 0, +impl ReadSetEntry { + /// The vShard whose versions validate this entry: its home, else its + /// collection's vShard. `None` for an unhomed entry whose collection + /// names no vShard. + pub fn validation_vshard(&self) -> Option { + match self.home { + Some(home) => Some(home), + None => { + nodedb_types::CollectionKey::from_qualified_str(self.database_id, &self.collection) + .ok() + .map(|key| key.vshard()) + } + } } } /// The observed read passed to [`record_read_set`]: the executed plan, the -/// responding shards' write-LSN watermarks, the read-version floor, and whether -/// a point read hit. +/// versions its responding cores reported, and whether a point read hit. pub struct ReadCapture<'a> { pub plan: &'a PhysicalPlan, - pub watermarks: &'a [(VShardId, Lsn)], - pub read_version_lsn: Lsn, + pub read_versions: &'a ReadVersions, pub found: bool, } /// Record a completed read into the session's transaction read-set. /// /// Transport-agnostic: every read post-dispatch seam calls this with the plan -/// that ran and the per-shard watermark(s) it observed. Records one -/// [`ReadSetEntry`] per `(vshard, watermark)` pair — a predicate read fanned -/// over N shards yields N entries, each carrying that shard's own watermark -/// LSN. A point read observes a single shard and yields one entry. +/// that ran and the versions it observed. +/// +/// A read of one collection records one entry that validates on the +/// collection's vShard at the version the read observed there. A vShard the +/// read reported no version of had no write the serving core knew of, so the +/// read observed it at `WriteVersion::ZERO`. A read that reported other +/// vShards too records one more entry per other reported vShard, homed +/// there. +/// +/// A read that names no collection observed every collection on each vShard +/// it reported. It records one entry per reported vShard, homed there. /// /// Guarded on the connection being inside a transaction block (the session /// write path drops the entries otherwise), so autocommit reads never touch -/// the read-set. Absent-key / empty-result reads MUST reach this with a -/// non-empty `watermarks` slice — a "not found" is a validatable observation. -/// -/// `read_version_lsn` is the read collection's per-collection write floor at -/// read time (a WAL LSN) — one scalar, since a read op -/// resolves to a single collection (joins collapse to one via -/// `extract_collection`). It stamps every entry produced here and is the SOUND -/// comparand cross-shard OCC validation consumes; the per-shard `watermarks` -/// still source the core-global `read_lsn` used by single-shard SI. +/// the read-set. Absent-key / empty-result reads are recorded too — a "not +/// found" is a validatable observation. /// /// `found` reports whether a point read observed a present row (`true` on a /// hit, `false` on a miss). It only affects document point reads — an absent @@ -211,13 +204,9 @@ pub async fn record_read_set( ) { let ReadCapture { plan, - watermarks, - read_version_lsn, + read_versions, found, } = capture; - if watermarks.is_empty() { - return; - } let engine = plan_engine(plan); let key = read_key_of(plan, found); @@ -231,45 +220,60 @@ pub async fn record_read_set( .get_current_database(session_id) .unwrap_or(DatabaseId::DEFAULT); - // Read-your-writes floor: raise the captured read-version to the session's - // OWN highest committed write-version for this collection. Without it, a - // read that observed a stale collection floor (0) before the session's own - // prior committed write was reflected on the serving core will, at - // cross-shard OCC validation, see that write's `coll_write_lsn` exceed the - // read-version and false-abort with a serialization failure on the session's - // OWN write. The floor is only ever raised by this session's own committed - // writes to this exact `(database, tenant, collection)`, so a concurrent - // OTHER-session write (higher `coll_write_lsn`) still exceeds the floor and - // still aborts — this removes only the self-abort. `read_lsn` (the per-shard - // watermark used by single-shard SI) is deliberately left untouched. - let own_write_version = - sessions.own_write_version(session_id, database_id, tenant_id, &collection); - let effective_read_version = read_version_lsn.max(own_write_version); - // The read validates on its collection's vShard, against the versions of - // the node that served it there. - let validation_vshard = - nodedb_types::CollectionKey::from_qualified_str(database_id, &collection) - .map(|key| key.vshard().as_u32()) - .unwrap_or_else(|_| watermarks[0].0.as_u32()); - let served_by = serving_node(state, validation_vshard); - - let entries: Vec = watermarks - .iter() - .map(|(_vshard, read_lsn)| ReadSetEntry { + let entry = |home: Option, observed: WriteVersion, validation: VShardId| { + // Read-your-writes floor: raise the observed version to the + // session's own highest committed write of this collection on the + // validation vShard. A read that ran before the serving replica + // applied the session's own prior write will otherwise abort on that + // write. Only this session's own writes raise the floor, so a + // concurrent write by another session still exceeds it and aborts. + let own = if collection.is_empty() { + WriteVersion::ZERO + } else { + sessions.own_write_version(session_id, database_id, tenant_id, &collection, validation) + }; + ReadSetEntry { engine, database_id, tenant_id, collection: collection.clone(), key: key.clone(), - read_lsn: *read_lsn, - read_version_lsn: effective_read_version, + read_version: observed.max(own), // Every entry captured here is a read the session issued inside its // own transaction, so the own-write exclusion applies to it. origin: ReadOrigin::Session, - home: None, - home_node: served_by, - }) - .collect(); + home, + } + }; + let entries: Vec = + match nodedb_types::CollectionKey::from_qualified_str(database_id, &collection) { + Ok(collection_key) if !collection.is_empty() => { + let validation = collection_key.vshard(); + let observed = read_versions.of(validation).unwrap_or_default(); + let mut entries = vec![entry(None, observed, validation)]; + entries.extend( + read_versions + .iter() + .map(|shard| VShardId::new(shard.vshard)) + .filter(|vshard| *vshard != validation) + .map(|vshard| { + let observed = read_versions.of(vshard).unwrap_or_default(); + entry(Some(vshard), observed, vshard) + }), + ); + entries + } + _ => read_versions + .iter() + .map(|shard| { + let vshard = VShardId::new(shard.vshard); + entry(Some(vshard), shard.version, vshard) + }) + .collect(), + }; + if entries.is_empty() { + return; + } sessions.record_read_entries(session_id, entries); @@ -432,7 +436,7 @@ mod tests { SessionId::LegacySocket(addr) => addr, SessionId::Connection(_) => unreachable!("legacy test identity"), }); - sessions.begin(session_id, Lsn::new(5), 0).expect("begin"); + sessions.begin(session_id, 0).expect("begin"); (sessions, session_id) } @@ -454,6 +458,18 @@ mod tests { (state, dir) } + /// The vShard a read of `collection` validates on. + fn home_of(collection: &str) -> VShardId { + nodedb_types::CollectionKey::from_qualified_str(DatabaseId::DEFAULT, collection) + .expect("collection key") + .vshard() + } + + /// `collection`'s vShard observed at log index `index`. + fn observed(collection: &str, index: u64) -> ReadVersions { + ReadVersions::single(home_of(collection), WriteVersion::logged(1, index)) + } + #[tokio::test] async fn point_read_records_point_key() { let (state, _dir) = test_state(); @@ -465,8 +481,7 @@ mod tests { TenantId::new(1), ReadCapture { plan: &kv_get("c", b"k1"), - watermarks: &[(VShardId::new(0), Lsn::new(7))], - read_version_lsn: Lsn::ZERO, + read_versions: &observed("c", 7), found: true, }, ) @@ -475,7 +490,9 @@ mod tests { assert_eq!(rs.len(), 1); assert_eq!(rs[0].engine, EngineTag::Kv); assert_eq!(rs[0].collection, "c"); - assert_eq!(rs[0].read_lsn, Lsn::new(7)); + assert_eq!(rs[0].read_version, WriteVersion::logged(1, 7)); + assert_eq!(rs[0].home, None); + assert_eq!(rs[0].validation_vshard(), Some(home_of("c"))); assert_eq!( rs[0].key, ReadKey::Point { @@ -497,8 +514,7 @@ mod tests { TenantId::new(1), ReadCapture { plan: &kv_batch_get("c"), - watermarks: &[(VShardId::new(0), Lsn::new(9))], - read_version_lsn: Lsn::ZERO, + read_versions: &observed("c", 9), found: true, }, ) @@ -509,11 +525,14 @@ mod tests { } #[tokio::test] - async fn multi_shard_read_records_one_entry_per_watermark() { + async fn a_read_records_every_vshard_it_reported() { let (state, _dir) = test_state(); let (sessions, a) = begun_session(); - // A predicate fanned over three cores records one entry per shard, each - // stamped with that shard's own watermark — NOT a single collapsed max. + // The collection's own vShard validates the unhomed entry. Another + // reported vShard validates an entry homed there. + let other = VShardId::new(home_of("c").as_u32() + 1); + let mut versions = observed("c", 11); + versions.note(other, WriteVersion::logged(1, 90)); record_read_set( &state, &sessions, @@ -521,21 +540,68 @@ mod tests { TenantId::new(1), ReadCapture { plan: &kv_batch_get("c"), - watermarks: &[ - (VShardId::new(0), Lsn::new(3)), - (VShardId::new(1), Lsn::new(11)), - (VShardId::new(2), Lsn::new(7)), - ], - read_version_lsn: Lsn::ZERO, + read_versions: &versions, found: true, }, ) .await; let rs = sessions.take_read_set(a); - assert_eq!(rs.len(), 3); - let mut lsns: Vec = rs.iter().map(|e| e.read_lsn.as_u64()).collect(); - lsns.sort_unstable(); - assert_eq!(lsns, vec![3, 7, 11]); + assert_eq!(rs.len(), 2); + assert_eq!(rs[0].home, None); + assert_eq!(rs[0].read_version, WriteVersion::logged(1, 11)); + assert_eq!(rs[1].home, Some(other)); + assert_eq!(rs[1].read_version, WriteVersion::logged(1, 90)); + assert_eq!(rs[1].collection, "c"); + } + + #[tokio::test] + async fn a_read_with_no_reported_version_observed_zero() { + let (state, _dir) = test_state(); + let (sessions, a) = begun_session(); + // The serving core knew no write of the vShard, so any write since + // the read moves it above the recorded version. + record_read_set( + &state, + &sessions, + a, + TenantId::new(1), + ReadCapture { + plan: &kv_get("c", b"k1"), + read_versions: &ReadVersions::new(), + found: false, + }, + ) + .await; + let rs = sessions.take_read_set(a); + assert_eq!(rs.len(), 1); + assert_eq!(rs[0].read_version, WriteVersion::ZERO); + } + + #[tokio::test] + async fn the_sessions_own_write_floors_the_read_version() { + let (state, _dir) = test_state(); + let (sessions, a) = begun_session(); + sessions.note_own_write( + a, + DatabaseId::DEFAULT, + TenantId::new(1), + "c", + &observed("c", 20), + ); + record_read_set( + &state, + &sessions, + a, + TenantId::new(1), + ReadCapture { + plan: &kv_get("c", b"k1"), + read_versions: &observed("c", 12), + found: true, + }, + ) + .await; + let rs = sessions.take_read_set(a); + assert_eq!(rs[0].read_version, WriteVersion::logged(1, 20)); } #[tokio::test] @@ -543,7 +609,7 @@ mod tests { let (state, _dir) = test_state(); let (sessions, a) = begun_session(); // A "not found" is a validatable phantom observation: the KV point entry - // is recorded (as `found = false`) at the current watermark. KV keeps the + // is recorded (as `found = false`) at the observed version. KV keeps the // precise byte key — the identity any future insert of that key reuses. record_read_set( &state, @@ -552,8 +618,7 @@ mod tests { TenantId::new(1), ReadCapture { plan: &kv_get("c", b"missing"), - watermarks: &[(VShardId::new(0), Lsn::new(5))], - read_version_lsn: Lsn::ZERO, + read_versions: &observed("c", 5), found: false, }, ) @@ -593,8 +658,7 @@ mod tests { TenantId::new(1), ReadCapture { plan: &doc_point_get("docs", 42), - watermarks: &[(VShardId::new(0), Lsn::new(7))], - read_version_lsn: Lsn::ZERO, + read_versions: &observed("docs", 7), found: true, }, ) @@ -623,8 +687,7 @@ mod tests { TenantId::new(1), ReadCapture { plan: &doc_point_get("docs", 999), - watermarks: &[(VShardId::new(0), Lsn::new(5))], - read_version_lsn: Lsn::ZERO, + read_versions: &observed("docs", 5), found: false, }, ) @@ -648,8 +711,7 @@ mod tests { TenantId::new(1), ReadCapture { plan: &kv_get("c", b"k1"), - watermarks: &[(VShardId::new(0), Lsn::new(7))], - read_version_lsn: Lsn::ZERO, + read_versions: &observed("c", 7), found: true, }, ) @@ -658,23 +720,35 @@ mod tests { } #[tokio::test] - async fn empty_watermarks_records_nothing() { + async fn a_read_of_no_collection_records_one_homed_entry_per_vshard() { let (state, _dir) = test_state(); let (sessions, a) = begun_session(); + // A read that names no collection observed every collection on each + // vShard it reported. + let plan = PhysicalPlan::Meta(nodedb_physical::physical_plan::MetaOp::HomeVersions { + probes: Vec::new(), + }); + let mut versions = ReadVersions::single(VShardId::new(3), WriteVersion::logged(1, 4)); + versions.note(VShardId::new(8), WriteVersion::logged(2, 6)); record_read_set( &state, &sessions, a, TenantId::new(1), ReadCapture { - plan: &kv_get("c", b"k1"), - watermarks: &[], - read_version_lsn: Lsn::ZERO, + plan: &plan, + read_versions: &versions, found: true, }, ) .await; - assert!(sessions.take_read_set(a).is_empty()); + let rs = sessions.take_read_set(a); + assert_eq!(rs.len(), 2); + assert_eq!(rs[0].home, Some(VShardId::new(3))); + assert_eq!(rs[0].read_version, WriteVersion::logged(1, 4)); + assert_eq!(rs[1].home, Some(VShardId::new(8))); + assert_eq!(rs[1].read_version, WriteVersion::logged(2, 6)); + assert!(rs.iter().all(|entry| entry.collection.is_empty())); } #[test] diff --git a/nodedb/src/control/server/shared/session/record_reads.rs b/nodedb/src/control/server/shared/session/record_reads.rs index f8340a545..bf7f6a01e 100644 --- a/nodedb/src/control/server/shared/session/record_reads.rs +++ b/nodedb/src/control/server/shared/session/record_reads.rs @@ -9,8 +9,8 @@ //! from its own single-collection scan plan and REAL observed read-version, so //! the commit-time OCC validator re-homes and revalidates each collection's //! vshard independently — while every other read records a single -//! collection-scoped entry from the executed plan and the responding shards' -//! watermarks. When captures are present the default single-collection entry is +//! collection-scoped entry from the executed plan and the versions its cores +//! reported. When captures are present the default single-collection entry is //! SKIPPED, because a `HashJoin` plan collapses to the left collection via //! `extract_collection` and would miss the build side entirely. This module //! hosts that decision so all transports funnel through one implementation @@ -19,7 +19,7 @@ use crate::bridge::envelope::PhysicalPlan; use crate::control::server::exchange::DistributedReadCapture; use crate::control::state::SharedState; -use crate::types::{Lsn, TenantId, VShardId}; +use crate::types::{ReadVersions, TenantId}; use super::connection::SessionId; use super::read_set::{ReadCapture, record_read_set}; @@ -27,22 +27,15 @@ use super::store::SessionStore; /// The observed reads produced by one dispatched response. /// -/// `plan` / `watermarks` / `read_version_lsn` / `found` describe the plain -/// single-collection observation (used when `distributed_reads` is empty). +/// `plan` / `read_versions` / `found` describe the plain single-collection +/// observation (used when `distributed_reads` is empty). /// `distributed_reads`, when non-empty, carries the per-collection captures of a /// distributed gather/shuffle read and takes precedence over the plain fields. -/// -/// `read_lsn_vshard` is the vshard stamped into each per-capture entry's -/// single-shard SI `read_lsn` slot (paired with [`Lsn::ZERO`], since the sound -/// cross-shard comparand is the capture's own `read_version_lsn`). It is -/// consulted only on the captures branch. pub struct ResponseReads<'a> { pub plan: &'a PhysicalPlan, - pub watermarks: &'a [(VShardId, Lsn)], - pub read_version_lsn: Lsn, + pub read_versions: &'a ReadVersions, pub found: bool, pub distributed_reads: &'a [DistributedReadCapture], - pub read_lsn_vshard: VShardId, } /// Record a dispatched response's reads into the session transaction read-set. @@ -68,8 +61,7 @@ pub async fn record_reads_for_response( tenant_id, ReadCapture { plan: &cap.scan_plan, - watermarks: &[(reads.read_lsn_vshard, Lsn::ZERO)], - read_version_lsn: cap.read_version_lsn, + read_versions: &cap.read_versions, found: false, }, ) @@ -83,8 +75,7 @@ pub async fn record_reads_for_response( tenant_id, ReadCapture { plan: reads.plan, - watermarks: reads.watermarks, - read_version_lsn: reads.read_version_lsn, + read_versions: reads.read_versions, found: reads.found, }, ) diff --git a/nodedb/src/control/server/shared/session/savepoint_ops.rs b/nodedb/src/control/server/shared/session/savepoint_ops.rs index 16a83560d..44c0deb43 100644 --- a/nodedb/src/control/server/shared/session/savepoint_ops.rs +++ b/nodedb/src/control/server/shared/session/savepoint_ops.rs @@ -308,7 +308,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }) }) @@ -344,7 +344,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:5203".parse().unwrap(); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).unwrap(); + store.begin(addr, 0).unwrap(); assert!(store.buffer_write(addr, staged_task(3))); let result = run_savepoint( @@ -386,7 +386,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:5201".parse().unwrap(); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).unwrap(); + store.begin(addr, 0).unwrap(); let tenant = TenantId::new(1); let dp = RecordingDp::default(); @@ -435,7 +435,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:5204".parse().unwrap(); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).unwrap(); + store.begin(addr, 0).unwrap(); let tenant = TenantId::new(1); let dp = RecordingDp::default(); assert!(store.buffer_write(addr, staged_task(2))); @@ -467,7 +467,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:5202".parse().unwrap(); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).unwrap(); + store.begin(addr, 0).unwrap(); let session = SessionId::from(&addr); let tenant = TenantId::new(1); let dp = RecordingDp::default(); diff --git a/nodedb/src/control/server/shared/session/served_reads.rs b/nodedb/src/control/server/shared/session/served_reads.rs deleted file mode 100644 index 625d59f55..000000000 --- a/nodedb/src/control/server/shared/session/served_reads.rs +++ /dev/null @@ -1,73 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! The node that served each vShard the running statement read. -//! -//! A read's versions are positions in the WAL of the node that served it, so -//! only that node's versions compare with them at commit. Every read dispatch -//! notes the vShard it read and the node that answered. The read-set capture -//! stamps each entry with that node (`ReadSetEntry::home_node`). A vShard two -//! nodes served in one statement has no one node, noted as `0`, and a commit -//! then treats its read as changed. -//! -//! The notes live in the connection scope and start empty with each -//! statement (`deadline::enter`). Outside a connection scope nothing is noted. - -use std::collections::HashMap; - -use super::conn_scope::with_scope; - -/// Note that `node` served a read of `vshard` in the running statement. -pub fn note(vshard: u32, node: u64) { - with_scope((), |scope| { - scope - .served_reads - .borrow_mut() - .entry(vshard) - .and_modify(|seen| { - if *seen != node { - *seen = 0; - } - }) - .or_insert(node); - }); -} - -/// The node that served the running statement's read of `vshard`: `Some(0)` -/// when two nodes did, `None` when no read of it was noted. -pub fn served_by(vshard: u32) -> Option { - with_scope(None, |scope| { - scope.served_reads.borrow().get(&vshard).copied() - }) -} - -/// Forget every note: a statement starts with no reads served. -pub(super) fn clear() { - with_scope((), |scope| scope.served_reads.borrow_mut().clear()); -} - -/// An empty note map for a fresh connection scope. -pub(super) fn empty() -> std::cell::RefCell> { - std::cell::RefCell::new(HashMap::new()) -} - -#[cfg(test)] -mod tests { - use super::super::conn_scope; - use super::*; - - #[tokio::test] - async fn a_vshard_two_nodes_served_has_no_node() { - conn_scope::scoped(async { - note(3, 1); - note(3, 1); - note(4, 1); - note(4, 2); - assert_eq!(served_by(3), Some(1)); - assert_eq!(served_by(4), Some(0)); - assert_eq!(served_by(5), None); - clear(); - assert_eq!(served_by(3), None); - }) - .await; - } -} diff --git a/nodedb/src/control/server/shared/session/state.rs b/nodedb/src/control/server/shared/session/state.rs index 1833f54f4..221d40da9 100644 --- a/nodedb/src/control/server/shared/session/state.rs +++ b/nodedb/src/control/server/shared/session/state.rs @@ -18,7 +18,9 @@ pub(crate) fn now_unix_ms() -> u64 { use crate::control::lease::QueryLeaseScope; use crate::event::cdc::CdcOffset; -use crate::types::{DatabaseId, Lsn, TenantId, TxnId, VShardId}; +use crate::types::{DatabaseId, TenantId, TxnId, VShardId}; + +use super::own_writes::OwnWriteKey; use nodedb_physical::physical_task::PhysicalTask; /// One entry on the transaction's savepoint stack. @@ -146,10 +148,6 @@ pub struct ConnSession { /// rejected a line has an entry. COMMIT compares these counts with its /// authoritative resolve. pub tx_ts_preview_rejected: BTreeMap, - /// Snapshot LSN captured at BEGIN for snapshot isolation. - /// All reads within the transaction see data as of this LSN. - /// Concurrent writes after this point are invisible to the transaction. - pub tx_snapshot_lsn: Option, /// Snapshot epoch captured at BEGIN: the last globally-applied Calvin epoch, /// read from `CalvinLocalState::last_applied_epoch`. The cross-shard-valid /// version anchor for the transaction (0 in single-node / no-Calvin). `None` @@ -166,11 +164,11 @@ pub struct ConnSession { /// mark/rewind must fan over ALL of them. Ordered (BTree) for deterministic /// teardown. Empty until the first staged write. pub tx_vshards: BTreeSet, - /// Read-set: LSN-versioned, predicate-aware entries for write conflict + /// Read-set: versioned, predicate-aware entries for write conflict /// detection, captured on the shared read seam by every transport. At /// COMMIT, each entry is checked — if the entry's collection has a current - /// write-LSN past `read_lsn`, a concurrent write occurred and the - /// transaction is rejected with SERIALIZATION_FAILURE. + /// version past `read_version` on its vShard, a concurrent write occurred + /// and the transaction is rejected with SERIALIZATION_FAILURE. pub tx_read_set: Vec, /// Distinct vShards this transaction took a SHARED read reservation on. A /// hot-key read reserves under the transaction's single `tx_reservation_owner` @@ -241,17 +239,14 @@ pub struct ConnSession { /// is idle-eligible only when this is zero — a legitimately long-running /// statement (in flight) must never be idle-killed. pub in_flight: AtomicU32, - /// Highest committed write-version this session has observed for each - /// `(database, tenant, collection)` it has written, keyed identically to - /// the read/write namespace. Used to floor a later transaction's captured - /// `read_version_lsn` at the session's OWN prior committed writes — a - /// read-your-writes floor that removes cross-shard OCC self-aborts on a - /// collection the session itself last wrote, without ever masking a - /// concurrent OTHER-session write (whose higher `coll_write_lsn` still - /// exceeds the floor). Persists for the life of the session — a prior - /// autocommit write must still floor a later transaction's read — and is - /// therefore NOT cleared at transaction boundaries. - pub own_write_versions: HashMap<(DatabaseId, TenantId, String), Lsn>, + /// Highest committed write version this session holds for each + /// `(database, tenant, collection, vShard)` it has written. It floors a + /// later transaction's captured read version at the session's own prior + /// committed writes, so a read never aborts on the session's own write. A + /// concurrent write by another session has a higher version and still + /// aborts. It persists for the life of the session, so a prior autocommit + /// write floors a later transaction's read too. + pub own_write_versions: HashMap, /// Database- and tenant-scoped connection slots this session holds. /// /// pgwire takes them during the startup handshake, once the bound database @@ -321,7 +316,6 @@ impl ConnSession { tx_lease_scopes: Vec::new(), tx_body_tasks: BTreeSet::new(), tx_ts_preview_rejected: BTreeMap::new(), - tx_snapshot_lsn: None, tx_snapshot_epoch: None, tx_id: None, tx_vshards: BTreeSet::new(), diff --git a/nodedb/src/control/server/shared/session/store.rs b/nodedb/src/control/server/shared/session/store.rs index f0e49f772..e1f3a8d91 100644 --- a/nodedb/src/control/server/shared/session/store.rs +++ b/nodedb/src/control/server/shared/session/store.rs @@ -303,7 +303,6 @@ impl SessionStore { session.tx_lease_scopes.clear(); session.tx_body_tasks.clear(); session.tx_ts_preview_rejected.clear(); - session.tx_snapshot_lsn = None; session.tx_snapshot_epoch = None; session.tx_id = None; session.tx_vshards.clear(); @@ -412,13 +411,12 @@ mod tests { use std::sync::Arc; use crate::control::lease::QueryLeaseScope; - use crate::types::Lsn; let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:6011".parse().expect("address"); store.ensure_session(addr); let scope = Arc::new(QueryLeaseScope::empty()); - store.begin(addr, Lsn::new(1), 0).expect("begin"); + store.begin(addr, 0).expect("begin"); assert!(store.buffer_write(addr, task())); assert!(store.attach_tx_lease_scope_since(addr, 0, Arc::clone(&scope))); store.rollback(addr).expect("rollback"); @@ -428,7 +426,7 @@ mod tests { assert_eq!(session.tx_buffer.len(), session.tx_lease_scopes.len()); }); - store.begin(addr, Lsn::new(2), 0).expect("begin"); + store.begin(addr, 0).expect("begin"); assert!(store.buffer_write(addr, task())); assert!(store.attach_tx_lease_scope_since(addr, 0, scope)); store.reset_for_database_switch(addr, DatabaseId::new(2)); diff --git a/nodedb/src/control/server/shared/session/transaction/buffer.rs b/nodedb/src/control/server/shared/session/transaction/buffer.rs index 6d41d559a..cda637a0b 100644 --- a/nodedb/src/control/server/shared/session/transaction/buffer.rs +++ b/nodedb/src/control/server/shared/session/transaction/buffer.rs @@ -223,7 +223,7 @@ mod tests { use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; use super::super::super::store::SessionStore; - use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; + use crate::types::{DatabaseId, TenantId, VShardId}; fn ingest_task() -> PhysicalTask { PhysicalTask { @@ -254,7 +254,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:5311".parse().expect("addr"); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).expect("begin"); + store.begin(addr, 0).expect("begin"); assert!(store.buffer_write(addr, ingest_task())); store.note_ts_preview_rejected(addr, 0); diff --git a/nodedb/src/control/server/shared/session/transaction/lifecycle.rs b/nodedb/src/control/server/shared/session/transaction/lifecycle.rs index ef9a3216e..d8f51c17a 100644 --- a/nodedb/src/control/server/shared/session/transaction/lifecycle.rs +++ b/nodedb/src/control/server/shared/session/transaction/lifecycle.rs @@ -8,7 +8,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use nodedb_physical::physical_task::PhysicalTask; use crate::control::lease::QueryLeaseScope; -use crate::types::{Lsn, TxnId, VShardId}; +use crate::types::{TxnId, VShardId}; use super::super::connection::SessionId; use super::super::state::TransactionState; @@ -31,22 +31,19 @@ impl SessionStore { .unwrap_or(TransactionState::Idle) } - /// BEGIN — enter transaction block with snapshot isolation. + /// BEGIN — enter a transaction block. /// - /// Captures the current WAL LSN as the local snapshot point (single-shard - /// fast path) and the last globally-applied Calvin `snapshot_epoch` as the - /// cross-shard-valid version anchor. All reads within this transaction see - /// data as of this LSN. + /// Captures the last globally-applied Calvin `snapshot_epoch` as the + /// cross-shard version anchor. Commit validates each read by the write + /// version it observed, so BEGIN captures no WAL position. pub fn begin( &self, addr: impl Into, - current_lsn: Lsn, snapshot_epoch: u64, ) -> Result<(), &'static str> { self.write_session(addr, |session| match session.tx_state { TransactionState::Idle => { session.tx_state = TransactionState::InBlock; - session.tx_snapshot_lsn = Some(current_lsn); session.tx_snapshot_epoch = Some(snapshot_epoch); session.tx_read_set.clear(); session.tx_reservation_vshards.clear(); @@ -75,11 +72,6 @@ impl SessionStore { .unwrap_or(false) } - /// Get the snapshot LSN for the current transaction. - pub fn snapshot_lsn(&self, addr: impl Into) -> Option { - self.read_session(addr, |s| s.tx_snapshot_lsn)? - } - /// Get the cross-shard snapshot epoch for the current transaction. pub fn snapshot_epoch(&self, addr: impl Into) -> Option { self.read_session(addr, |s| s.tx_snapshot_epoch)? @@ -116,7 +108,6 @@ impl SessionStore { session.tx_body_tasks.clear(); session.tx_ts_preview_rejected.clear(); session.tx_state = TransactionState::Idle; - session.tx_snapshot_lsn = None; session.tx_snapshot_epoch = None; session.tx_id = None; session.tx_vshards.clear(); @@ -147,7 +138,6 @@ impl SessionStore { session.tx_ts_preview_rejected.clear(); session.pending_publishes.clear(); session.tx_state = TransactionState::Idle; - session.tx_snapshot_lsn = None; session.tx_snapshot_epoch = None; session.tx_id = None; session.tx_vshards.clear(); @@ -203,13 +193,13 @@ mod tests { assert_eq!(store.transaction_state(addr), TransactionState::Idle); - store.begin(addr, Lsn::new(1), 0).unwrap(); + store.begin(addr, 0).unwrap(); assert_eq!(store.transaction_state(addr), TransactionState::InBlock); store.commit(addr).unwrap(); assert_eq!(store.transaction_state(addr), TransactionState::Idle); - store.begin(addr, Lsn::new(1), 0).unwrap(); + store.begin(addr, 0).unwrap(); store.fail_transaction(addr); assert_eq!(store.transaction_state(addr), TransactionState::Failed); @@ -222,7 +212,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:6012".parse().expect("address"); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).expect("begin"); + store.begin(addr, 0).expect("begin"); let scope = Arc::new(QueryLeaseScope::empty()); assert!(store.buffer_write(addr, task())); diff --git a/nodedb/src/control/server/shared/session/transaction/savepoints.rs b/nodedb/src/control/server/shared/session/transaction/savepoints.rs index a4b352d7f..e07957fc0 100644 --- a/nodedb/src/control/server/shared/session/transaction/savepoints.rs +++ b/nodedb/src/control/server/shared/session/transaction/savepoints.rs @@ -140,7 +140,7 @@ mod tests { use crate::control::lease::QueryLeaseScope; use crate::control::server::shared::session::state::PendingOffsetCommit; - use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; + use crate::types::{DatabaseId, TenantId, VShardId}; fn task() -> PhysicalTask { PhysicalTask { @@ -160,7 +160,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:6010".parse().expect("address"); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).expect("begin"); + store.begin(addr, 0).expect("begin"); let scope = Arc::new(QueryLeaseScope::empty()); assert!(store.buffer_write(addr, task())); @@ -184,7 +184,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:6013".parse().expect("address"); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).expect("begin"); + store.begin(addr, 0).expect("begin"); let before = PendingOffsetCommit { database_id: DatabaseId::DEFAULT, @@ -221,7 +221,7 @@ mod tests { let store = SessionStore::new(); let addr: std::net::SocketAddr = "127.0.0.1:6014".parse().expect("address"); store.ensure_session(addr); - store.begin(addr, Lsn::new(1), 0).expect("begin"); + store.begin(addr, 0).expect("begin"); store.create_savepoint(addr, "sp".into(), 1, 0); store.fail_transaction(addr); assert_eq!(store.transaction_state(addr), TransactionState::Failed); diff --git a/nodedb/src/control/server/shared/statement_exec/dispatch.rs b/nodedb/src/control/server/shared/statement_exec/dispatch.rs index 771704626..d9fc36ff5 100644 --- a/nodedb/src/control/server/shared/statement_exec/dispatch.rs +++ b/nodedb/src/control/server/shared/statement_exec/dispatch.rs @@ -23,13 +23,11 @@ use crate::control::server::shared::clone_write::{ CloneCheckedOutcome, InterceptAndAuthorizeParams, intercept_and_authorize, }; use crate::control::state::SharedState; -use crate::types::{Lsn, TraceId, VShardId}; +use crate::types::TraceId; /// One dispatched task's answer. pub(crate) struct DispatchedTask { pub(crate) response: Response, - /// The watermark of each shard a single-node fan gather read. - pub(crate) shard_watermarks: Vec<(VShardId, Lsn)>, /// The reads a distributed gather captured. pub(crate) distributed_reads: Vec, } @@ -38,7 +36,6 @@ impl DispatchedTask { fn answered(response: Response) -> Self { Self { response, - shard_watermarks: Vec::new(), distributed_reads: Vec::new(), } } @@ -168,10 +165,9 @@ pub(crate) async fn dispatch_statement_task( linearizable: true, }; match resolve_and_materialize(state, identity, task.plan, scope).await? { - Resolved::Gathered(response, shard_watermarks, distributed_reads) => { + Resolved::Gathered(response, _watermarks, distributed_reads) => { return Ok(DispatchedTask { response, - shard_watermarks, distributed_reads, }); } diff --git a/nodedb/src/control/server/shared/statement_exec/task_loop.rs b/nodedb/src/control/server/shared/statement_exec/task_loop.rs index cda8c921d..0abfd8ae8 100644 --- a/nodedb/src/control/server/shared/statement_exec/task_loop.rs +++ b/nodedb/src/control/server/shared/statement_exec/task_loop.rs @@ -54,6 +54,9 @@ pub(crate) struct StatementExec<'a> { /// The sequence access the statement's Control-Plane computed columns /// read: the registry, plus the session's `currval` map when it has one. pub(crate) sequences: &'a SessionSequenceAccess<'a>, + /// The client's session, which notes its own committed writes. `None` + /// for a transport with no session. + pub(crate) client_session: Option>, } impl StatementExec<'_> { @@ -201,31 +204,32 @@ pub(crate) async fn run_statement_loop( continue; } - let task_vshard = task.vshard_id; let task_database_id = task.database_id; let DispatchedTask { response: task_response, - shard_watermarks, distributed_reads, } = dispatch_statement_task(state, exec.identity, task) .await .map_err(StatementError::Error)?; - // Track reads for snapshot-isolation / cross-shard conflict detection - // into the statement's transaction. Recorded BEFORE the error - // short-circuit so an absent-key point read (a `NotFound` from the - // Data Plane) is captured too: a "not found" is a validatable phantom - // observation. A multi-core fan read records one entry per - // participating shard from the gather's per-shard watermarks. A - // single read falls back to its one watermark. + // Track reads for commit-time conflict detection into the + // statement's transaction. Recorded BEFORE the error short-circuit so + // an absent-key point read (a `NotFound` from the Data Plane) is + // captured too: a "not found" is a validatable phantom observation. + // A committed write floors the session's later reads at the + // versions it stamped. + if let Some(client) = &exec.client_session { + client.sessions.note_own_write_response( + client.session_id, + task_database_id, + exec.tenant_id(), + &plan, + &task_response, + ); + } let records_read = task_response.status == Status::Ok || task_response.error_code.as_deref() == Some(&ErrorCode::NotFound); if records_read && let Some(txn) = txn { - let watermarks = if shard_watermarks.is_empty() { - vec![(task_vshard, task_response.watermark_lsn)] - } else { - shard_watermarks - }; record_reads_for_response( state, txn.sessions, @@ -233,11 +237,9 @@ pub(crate) async fn run_statement_loop( exec.tenant_id(), ResponseReads { plan: &plan, - watermarks: &watermarks, - read_version_lsn: task_response.read_version_lsn, + read_versions: &task_response.read_versions, found: task_response.status == Status::Ok, distributed_reads: &distributed_reads, - read_lsn_vshard: task_vshard, }, ) .await; diff --git a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs index 9a085dae9..2a878ca9e 100644 --- a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs +++ b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs @@ -423,8 +423,7 @@ mod tests { use nodedb_types::timeseries::continuous_agg::{ContinuousAggregateDef, RefreshPolicy}; use nodedb_types::vector_distance::DistanceMetric; use nodedb_types::{ - DatabaseId, Lsn, QualifiedCollection, Surrogate, SystemTimeScope, TenantId, - VectorAnnOptions, + DatabaseId, QualifiedCollection, Surrogate, SystemTimeScope, TenantId, VectorAnnOptions, }; use std::collections::BTreeMap; @@ -1844,6 +1843,7 @@ mod tests { replace_mode: false, collections_to_clear: Vec::new(), group_vshards: Vec::new(), + version_floor: Vec::new(), }), PhysicalPlan::Meta(MetaOp::PurgeTenant { tenant_id: 1 }), PhysicalPlan::Meta(MetaOp::UnregisterCollection { @@ -1918,9 +1918,8 @@ mod tests { engine: EngineTag::Kv, collection: QualifiedCollection::new(DatabaseId::DEFAULT, "c").to_string(), key: ReadKeyIdent::Predicate, - read_lsn: Lsn::ZERO, + read_version: nodedb_types::WriteVersion::ZERO, home_vshard: None, - served_by: 0, }], body_plans: Vec::new(), }), diff --git a/nodedb/src/control/server/shared/write_admission/route.rs b/nodedb/src/control/server/shared/write_admission/route.rs index cb3806279..c20905d70 100644 --- a/nodedb/src/control/server/shared/write_admission/route.rs +++ b/nodedb/src/control/server/shared/write_admission/route.rs @@ -62,7 +62,7 @@ pub fn bare_ok_response(request_id: RequestId) -> Response { watermark_lsn: crate::types::Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/server/shuffle/aggregator_hook.rs b/nodedb/src/control/server/shuffle/aggregator_hook.rs index c55267f1e..6119ef186 100644 --- a/nodedb/src/control/server/shuffle/aggregator_hook.rs +++ b/nodedb/src/control/server/shuffle/aggregator_hook.rs @@ -184,6 +184,7 @@ impl RegistryShuffleAggregator { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/control/server/shuffle/consumer_hook.rs b/nodedb/src/control/server/shuffle/consumer_hook.rs index 15a1d3d3d..e4cc0f988 100644 --- a/nodedb/src/control/server/shuffle/consumer_hook.rs +++ b/nodedb/src/control/server/shuffle/consumer_hook.rs @@ -187,6 +187,7 @@ impl RegistryShuffleConsumer { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/control/server/shuffle/fanout.rs b/nodedb/src/control/server/shuffle/fanout.rs index 730c3346e..e5ab2de01 100644 --- a/nodedb/src/control/server/shuffle/fanout.rs +++ b/nodedb/src/control/server/shuffle/fanout.rs @@ -78,13 +78,12 @@ pub struct ShuffleFanoutSink { keys: Vec, /// `part -> PartState`, one entry per part in `0..num_parts`. parts: HashMap, - /// Max per-collection read-version LSN observed across every scanned frame - /// fed to this sink (each frame's scanned collection `coll_write_lsn` at read - /// time). A producer scans exactly one collection, so this single max-folded - /// value is that collection's observed read version — the sound comparand the - /// coordinator uses for cross-shard OCC read validation of an in-transaction - /// distributed aggregate. - max_read_version_lsn: u64, + /// The read versions observed across every scanned frame fed to this sink, + /// one per vShard. A producer scans exactly one collection, so the fold + /// holds that collection's observed versions: the comparand the + /// coordinator uses for cross-shard OCC read validation of an + /// in-transaction distributed aggregate. + read_versions: crate::types::ReadVersions, } /// Parameters for [`ShuffleFanoutSink::new`]. @@ -148,16 +147,16 @@ impl ShuffleFanoutSink { producer_count, keys, parts, - max_read_version_lsn: 0, + read_versions: crate::types::ReadVersions::new(), } } - /// The max per-collection read-version LSN observed across every scanned - /// frame fed to this sink (`0` if no frame carried one). The producer hook - /// reads this after the streaming scan completes and reports it on the + /// The read versions observed across every scanned frame fed to this + /// sink (empty if no frame carried one). The producer hook reads them + /// after the streaming scan completes and reports them on the /// `ShuffleProduceResponse` so the coordinator can validate the read. - pub fn observed_read_version_lsn(&self) -> u64 { - self.max_read_version_lsn + pub fn observed_read_versions(&self) -> &crate::types::ReadVersions { + &self.read_versions } /// The `ShufflePushRequest` opener for `part` (shared shape for remote + @@ -412,16 +411,17 @@ impl nodedb_cluster::ChunkSink for &mut ShuffleFanoutSink { &mut self, payload: Vec, _watermark_lsn: u64, - read_version_lsn: u64, + read_versions: Vec, ) -> nodedb_cluster::Result<()> { - // The streaming executor calls this per scan batch. Max-fold this frame's - // per-collection read version so the producer can report the observed - // read version for cross-shard OCC validation (NOT the core-global - // watermark). Then hash-partition the batch and buffer/flush per part. A + // The streaming executor calls this per scan batch. Fold this frame's + // read versions so the producer can report what it observed for + // cross-shard OCC validation (NOT the core-global watermark). Then + // hash-partition the batch and buffer/flush per part. A // routing/flush failure is surfaced as a cluster error so the streaming // executor terminates the scan; the producer hook then `finish`es the // fan-out with the error so consumers fail fast (never a silent drop). - self.max_read_version_lsn = self.max_read_version_lsn.max(read_version_lsn); + self.read_versions + .merge(&crate::types::ReadVersions::from_wire(&read_versions)); self.route_chunk(payload) .await .map_err(|e| nodedb_cluster::ClusterError::Storage { diff --git a/nodedb/src/control/server/shuffle/producer_hook.rs b/nodedb/src/control/server/shuffle/producer_hook.rs index 9be9b4442..06c622530 100644 --- a/nodedb/src/control/server/shuffle/producer_hook.rs +++ b/nodedb/src/control/server/shuffle/producer_hook.rs @@ -56,9 +56,8 @@ impl RegistryShuffleProducer { } /// Run the produce, returning a [`ShuffleProduceResponse`] whose `error` is a - /// terminal error (or `None` on clean produce) and whose `read_version_lsn` is - /// the max per-collection read version the local scan observed (`0` on - /// failure). + /// terminal error (or `None` on clean produce) and whose `read_versions` are + /// the read versions the local scan observed (empty on failure). /// /// Factored out of the trait method so the trait impl stays a thin shim and /// every early-exit error path is `?`-propagated here, then mapped to the @@ -72,7 +71,7 @@ impl RegistryShuffleProducer { message: "shuffle produce requires a cluster transport (single-node mode)" .into(), }), - read_version_lsn: 0, + read_versions: Vec::new(), }; }; @@ -129,12 +128,12 @@ impl RegistryShuffleProducer { // (validation reject, decode, deadline, or data-plane stream error). let scan_outcome = executor.execute_plan_streaming(exec_req, &mut sink).await; - // Capture the max per-collection read version the scan observed BEFORE - // `finish` consumes the sink. On a clean produce this is the scanned - // collection's `coll_write_lsn` at read time — the comparand the - // coordinator max-folds across producers for cross-shard OCC read - // validation of an in-transaction distributed aggregate. - let observed_read_version = sink.observed_read_version_lsn(); + // Capture the read versions the scan observed BEFORE `finish` consumes + // the sink. On a clean produce they are the scanned collection's + // versions at read time: the comparand the coordinator folds across + // producers for cross-shard OCC read validation of an in-transaction + // distributed aggregate. + let observed_read_versions = sink.observed_read_versions().to_wire(); // Finalize the fan-out: flush residuals (clean path) and `End` EVERY part // for this side — with the scan error if any — so each receiver's barrier @@ -148,20 +147,21 @@ impl RegistryShuffleProducer { code: 0, message: format!("shuffle produce fan-out finalize failed: {e}"), })), - read_version_lsn: 0, + read_versions: Vec::new(), }; } - // On a failed scan the read version is meaningless — report 0 (mirroring - // `ExecuteResponse::err`). On a clean produce report the observed max. - let read_version_lsn = if scan_outcome.is_none() { - observed_read_version + // On a failed scan the read versions are meaningless: report none + // (mirroring `ExecuteResponse::err`). On a clean produce report what + // the scan observed. + let read_versions = if scan_outcome.is_none() { + observed_read_versions } else { - 0 + Vec::new() }; ShuffleProduceResponse { error: scan_outcome, - read_version_lsn, + read_versions, } } } diff --git a/nodedb/src/control/server/sync/raft_dispatch/durability_test_support.rs b/nodedb/src/control/server/sync/raft_dispatch/durability_test_support.rs index 35cedebc2..4e36ec249 100644 --- a/nodedb/src/control/server/sync/raft_dispatch/durability_test_support.rs +++ b/nodedb/src/control/server/sync/raft_dispatch/durability_test_support.rs @@ -18,7 +18,7 @@ use crate::control::security::audit::NoopAuditEmitter; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::shared::authorization::{AuthorizedTask, authorize_task_set}; use crate::control::state::SharedState; -use crate::types::{DatabaseId, Lsn, TenantId, VShardId}; +use crate::types::{DatabaseId, ReadVersions, TenantId, VShardId}; use crate::wal::WalManager; pub(super) const COLLECTION: &str = "docs"; @@ -48,7 +48,7 @@ pub(super) fn fixture() -> (Arc, CoreChannelDataSide, tempfile::Tem /// answers each proposal with the payload `applied`. pub(super) fn applying_proposer() -> Arc { Arc::new(|_vshard, _key, _data, _deadline| { - Box::pin(async { Ok((b"applied".to_vec(), Lsn::ZERO)) }) + Box::pin(async { Ok((b"applied".to_vec(), ReadVersions::new())) }) }) } diff --git a/nodedb/src/control/server/sync/raft_dispatch/propose.rs b/nodedb/src/control/server/sync/raft_dispatch/propose.rs index 0461af376..f2001f6ee 100644 --- a/nodedb/src/control/server/sync/raft_dispatch/propose.rs +++ b/nodedb/src/control/server/sync/raft_dispatch/propose.rs @@ -109,9 +109,9 @@ pub(crate) async fn propose_sync_write( for (attempt, backoff_ms) in BACKOFF_MS.iter().enumerate() { match proposer(vshard_id, idempotency_key, data.clone(), deadline).await { - // The committed log index rides alongside the payload; the sync-ack - // path only needs the payload bytes. - Ok((p, _committed_version)) => { + // The write's versions ride alongside the payload. The sync-ack + // path needs only the payload bytes. + Ok((p, _write_versions)) => { payload = Some(p); break; } diff --git a/nodedb/src/control/server/sync/raft_dispatch/response.rs b/nodedb/src/control/server/sync/raft_dispatch/response.rs index 36c16c3d0..8be27caf8 100644 --- a/nodedb/src/control/server/sync/raft_dispatch/response.rs +++ b/nodedb/src/control/server/sync/raft_dispatch/response.rs @@ -73,7 +73,7 @@ fn ok_response(state: &SharedState, payload: Vec) -> Response { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/shutdown/data_plane.rs b/nodedb/src/control/shutdown/data_plane.rs index a8b399046..ccf3fc4fa 100644 --- a/nodedb/src/control/shutdown/data_plane.rs +++ b/nodedb/src/control/shutdown/data_plane.rs @@ -274,6 +274,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), } } @@ -288,7 +289,7 @@ mod tests { watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/state/calvin_ack_result.rs b/nodedb/src/control/state/calvin_ack_result.rs index 980484068..965e814f8 100644 --- a/nodedb/src/control/state/calvin_ack_result.rs +++ b/nodedb/src/control/state/calvin_ack_result.rs @@ -165,7 +165,7 @@ mod tests { watermark_lsn: crate::types::Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/control/state/test_core.rs b/nodedb/src/control/state/test_core.rs index bc0d95f5f..ff8007515 100644 --- a/nodedb/src/control/state/test_core.rs +++ b/nodedb/src/control/state/test_core.rs @@ -30,7 +30,7 @@ pub(crate) async fn acknowledge_every_request( watermark_lsn: Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, }) diff --git a/nodedb/src/control/system_txn/scope.rs b/nodedb/src/control/system_txn/scope.rs index 2e76e5bd4..3df345650 100644 --- a/nodedb/src/control/system_txn/scope.rs +++ b/nodedb/src/control/system_txn/scope.rs @@ -28,10 +28,6 @@ impl SystemTxnScope { sessions.ensure_session(addr); let session_id = SessionId::from(addr); - let snapshot_lsn = { - let next = state.wal.next_lsn(); - crate::types::Lsn::new(next.as_u64().saturating_sub(1)) - }; let snapshot_epoch = state .calvin .last_applied_epoch @@ -41,7 +37,7 @@ impl SystemTxnScope { // around the transaction, and COMMIT applies it with the writes. crate::control::server::shared::session::ddl_buffer::activate(); sessions - .begin(session_id, snapshot_lsn, snapshot_epoch) + .begin(session_id, snapshot_epoch) .map_err(|detail| crate::Error::BadRequest { detail: detail.to_owned(), })?; diff --git a/nodedb/src/control/system_txn/tests.rs b/nodedb/src/control/system_txn/tests.rs index 363887b31..94631f368 100644 --- a/nodedb/src/control/system_txn/tests.rs +++ b/nodedb/src/control/system_txn/tests.rs @@ -112,7 +112,7 @@ async fn answer_all(state: Arc, mut side: CoreChannelDataSide, stop watermark_lsn: Lsn::ZERO, error_code, stage_vote: None, - read_version_lsn: Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }; side.response_tx diff --git a/nodedb/src/control/vshard_admission.rs b/nodedb/src/control/vshard_admission.rs index 5c6923c03..60375a64f 100644 --- a/nodedb/src/control/vshard_admission.rs +++ b/nodedb/src/control/vshard_admission.rs @@ -305,7 +305,12 @@ mod tests { use tokio::sync::{Barrier, Notify}; use super::*; - use crate::types::Lsn; + use crate::types::ReadVersions; + + /// The versions a test proposer answers proposal `key` with. + fn applied_at(key: u64) -> ReadVersions { + ReadVersions::single(VShardId::new(0), nodedb_types::WriteVersion::logged(0, key)) + } fn test_deadline() -> tokio::time::Instant { tokio::time::Instant::now() + std::time::Duration::from_secs(30) @@ -536,7 +541,7 @@ mod tests { entered.notify_one(); release.notified().await; active.fetch_sub(1, Ordering::SeqCst); - Ok((data, Lsn::new(key))) + Ok((data, applied_at(key))) }) }) }; @@ -557,13 +562,13 @@ mod tests { release.notify_one(); assert_eq!( first.await.expect("first joins").expect("first success"), - (vec![1], Lsn::new(11)) + (vec![1], applied_at(11)) ); entered.notified().await; release.notify_one(); assert_eq!( second.await.expect("second joins").expect("second success"), - (vec![2], Lsn::new(12)) + (vec![2], applied_at(12)) ); assert_eq!(maximum.load(Ordering::SeqCst), 1); } @@ -577,7 +582,7 @@ mod tests { let proposed = Arc::clone(&proposed); Box::pin(async move { proposed.lock().expect("proposed log").push(key); - Ok((data, Lsn::new(key))) + Ok((data, applied_at(key))) }) }) }; @@ -658,11 +663,11 @@ mod tests { .await .expect("second joins") .expect("second proposes"), - (vec![22], Lsn::new(22)) + (vec![22], applied_at(22)) ); assert_eq!( third.await.expect("third joins").expect("third proposes"), - (vec![23], Lsn::new(23)) + (vec![23], applied_at(23)) ); assert_eq!( *proposed.lock().expect("proposed log"), @@ -696,7 +701,7 @@ mod tests { detail: format!("test apply gate closed: {e}"), })? .forget(); - Ok((data, Lsn::new(key))) + Ok((data, applied_at(key))) }), }) }) @@ -744,10 +749,10 @@ mod tests { first.await.expect("first joins").expect("first applies"), second.await.expect("second joins").expect("second applies"), ]; - results.sort_by_key(|(_, lsn)| lsn.as_u64()); + results.sort_by_key(|(_, versions)| versions.of(VShardId::new(0))); assert_eq!( results, - vec![(vec![1], Lsn::new(11)), (vec![2], Lsn::new(12))] + vec![(vec![1], applied_at(11)), (vec![2], applied_at(12))] ); assert_eq!(sequencer.slots[5].capacity.available_permits(), 4); } diff --git a/nodedb/src/control/wal_replication/mod.rs b/nodedb/src/control/wal_replication/mod.rs index 184ddb2e1..bf0e1cd61 100644 --- a/nodedb/src/control/wal_replication/mod.rs +++ b/nodedb/src/control/wal_replication/mod.rs @@ -23,9 +23,10 @@ pub(crate) use propose::{ }; pub use replicable_write::ReplicableWrite; pub use types::{ - AppliedWait, AsyncRaftProposer, AsyncRaftSubmit, CollectionIncarnation, ConstraintChangeOp, - ProposedAt, ProposedWrite, RaftAppliedIndexSink, RaftCompactor, RaftProposer, ReplicatedEntry, - ReplicatedEventSource, ReplicatedIdentity, ReplicatedSumTarget, ReplicatedWrite, + AppliedOutput, AppliedWait, AsyncRaftProposer, AsyncRaftSubmit, CollectionIncarnation, + ConstraintChangeOp, ProposedAt, ProposedWrite, RaftAppliedIndexSink, RaftCompactor, + RaftProposer, ReplicatedEntry, ReplicatedEventSource, ReplicatedIdentity, ReplicatedSumTarget, + ReplicatedWrite, }; pub use crate::control::distributed_applier::{ diff --git a/nodedb/src/control/wal_replication/propose.rs b/nodedb/src/control/wal_replication/propose.rs index 13cbba08a..6f85d6503 100644 --- a/nodedb/src/control/wal_replication/propose.rs +++ b/nodedb/src/control/wal_replication/propose.rs @@ -54,13 +54,11 @@ pub(crate) fn stamp_collection_incarnations( Ok(()) } -/// Propose `entry` via `proposer` and return the Data Plane apply payload bytes -/// together with the write's per-collection version (as an -/// [`crate::types::Lsn`]): the written collection's `coll_write_lsn` after the -/// write, stamped by the applying replica from the WAL LSN it minted for the -/// entry's redo record. `Lsn::ZERO` when the write's plan names no single user -/// collection. See [`AsyncRaftProposer`] for why this is a WAL LSN and never the -/// Raft log index. +/// Propose `entry` via `proposer` and return the Data Plane apply payload +/// bytes and the versions the write stamped. Each version is the data-group +/// log position of the entry that applied the write, so it is the same on +/// every replica. A write whose plan names no single user collection stamps +/// none. /// /// An edge write is not proposed. It runs as a Calvin transaction /// (`planner::calvin::edge_sequencing`), and its applied payload and read @@ -85,13 +83,13 @@ pub(crate) async fn propose_replicated_entry( proposer: &Arc, mut entry: ReplicatedEntry, deadline: tokio::time::Instant, -) -> crate::Result<(Vec, crate::types::Lsn)> { +) -> crate::Result { // An edge write runs as a Calvin transaction, never as a data-group // entry, so every edge version of a collection takes a Calvin ordinal. if let Some(response) = crate::control::planner::calvin::sequence_replicated_edge_write(state, &entry).await? { - return Ok((response.payload.to_vec(), response.read_version_lsn)); + return Ok((response.payload.to_vec(), response.read_versions)); } // The write's commit instant. Stamped once, before the first propose, so // every re-proposal and every replica's apply carries the same value. diff --git a/nodedb/src/control/wal_replication/transaction_redo/propose.rs b/nodedb/src/control/wal_replication/transaction_redo/propose.rs index 988d067c0..fc1c608c7 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/propose.rs +++ b/nodedb/src/control/wal_replication/transaction_redo/propose.rs @@ -15,7 +15,6 @@ use crate::control::wal_replication::encode::{ RedoEntryTarget, RedoProposal, session_redo_proposal, }; use crate::control::wal_replication::propose_replicated_entry; -use crate::types::Lsn; use crate::wal::RedoStreamId; use super::abandon::propose_abandon; @@ -30,7 +29,7 @@ pub(crate) async fn propose_transaction_redo( target: RedoTarget, payload: &TransactionRedoPayload, deadline: tokio::time::Instant, -) -> crate::Result<(Vec, Lsn)> { +) -> crate::Result { let proposer = state.async_raft_proposer()?; let entry_target = RedoEntryTarget { tenant_id: target.tenant_id, diff --git a/nodedb/src/control/wal_replication/types/aliases.rs b/nodedb/src/control/wal_replication/types/aliases.rs index 23fa6d5bd..b9bbd5def 100644 --- a/nodedb/src/control/wal_replication/types/aliases.rs +++ b/nodedb/src/control/wal_replication/types/aliases.rs @@ -13,15 +13,9 @@ pub type RaftProposer = /// Type alias for the asynchronous Raft propose callback with leader forwarding. /// /// Takes `(vshard_id, idempotency_key, serialized_entry, deadline)` and returns, on -/// success, the Data Plane apply payload bytes together with the write's -/// per-collection version: the written collection's `coll_write_lsn` AFTER the -/// write, as the replica that applied the entry recorded it. It is a WAL LSN, -/// minted by the apply path's own WAL append — NOT the Raft log index, which is -/// a per-group counter sharing no scale with the WAL LSNs the version index is -/// otherwise fed. Callers that need the write's version (e.g. a -/// read-your-writes floor for cross-shard OCC, which validates in the WAL-LSN -/// domain) read it here without a second lookup; it is `Lsn::ZERO` for a write -/// whose plan names no single user collection, which records no such version. +/// success, an [`AppliedOutput`]: the Data Plane apply payload bytes and the +/// write's versions. A write's version is the data-group log position of the +/// entry that applied it, so every replica records the same version. /// The `idempotency_key` matches the one embedded in the /// serialized `ReplicatedEntry`; the proposer registers the tracker waiter with /// this key so apply-side mismatch detection can surface `RetryableLeaderChange` @@ -38,13 +32,17 @@ pub type AsyncRaftProposer = dyn Fn( tokio::time::Instant, ) -> std::pin::Pin< Box< - dyn std::future::Future< - Output = std::result::Result<(Vec, crate::types::Lsn), crate::Error>, - > + Send, + dyn std::future::Future> + + Send, >, > + Send + Sync; +/// What this node's apply of a proposed write returns: the Data Plane apply +/// payload and the versions the write stamped, one per written vShard. A +/// write that names no single user collection stamps none. +pub type AppliedOutput = (Vec, crate::types::ReadVersions); + /// Where a proposed entry landed in its data group's Raft log. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ProposedAt { @@ -55,11 +53,7 @@ pub struct ProposedAt { /// The wait for this node's apply of a proposed entry: the result an /// [`AsyncRaftProposer`] returns. pub type AppliedWait = std::pin::Pin< - Box< - dyn std::future::Future< - Output = std::result::Result<(Vec, crate::types::Lsn), crate::Error>, - > + Send, - >, + Box> + Send>, >; /// A proposal the group's leader accepted into its log. diff --git a/nodedb/src/control/wal_replication/types/mod.rs b/nodedb/src/control/wal_replication/types/mod.rs index d52e6a660..1080445d2 100644 --- a/nodedb/src/control/wal_replication/types/mod.rs +++ b/nodedb/src/control/wal_replication/types/mod.rs @@ -27,7 +27,7 @@ mod transaction_redo_wire; mod wire_shapes; pub use aliases::{ - AppliedWait, AsyncRaftProposer, AsyncRaftSubmit, ProposedAt, ProposedWrite, + AppliedOutput, AppliedWait, AsyncRaftProposer, AsyncRaftSubmit, ProposedAt, ProposedWrite, RaftAppliedIndexSink, RaftCompactor, RaftProposer, }; pub use redo_chunk_wire::{RedoBody, RedoContent}; diff --git a/nodedb/src/control/write_resolve/propose.rs b/nodedb/src/control/write_resolve/propose.rs index e06b3224f..16e7b697a 100644 --- a/nodedb/src/control/write_resolve/propose.rs +++ b/nodedb/src/control/write_resolve/propose.rs @@ -44,7 +44,7 @@ pub(super) async fn propose_resolved( let deadline = crate::control::wal_replication::statement_propose_deadline(state); match propose_replicated_entry(state, proposer, entry, deadline).await { - Ok((payload, write_version)) => { + Ok((payload, write_versions)) => { let request_id = RequestId::new(state.request_id_counter.fetch_add(1, Ordering::Relaxed)); let response = Response { @@ -53,10 +53,11 @@ pub(super) async fn propose_resolved( attempt: 1, partial: false, payload: payload.into(), - watermark_lsn: write_version, + // A write carries no read watermark. + watermark_lsn: crate::types::Lsn::ZERO, error_code: None, stage_vote: None, - read_version_lsn: write_version, + read_versions: write_versions, write_set: Vec::new(), }; Ok(ProposeOutcome::Applied(response)) diff --git a/nodedb/src/data/executor/array_checkpoint.rs b/nodedb/src/data/executor/array_checkpoint.rs index 1bd1f445c..ff234e7bf 100644 --- a/nodedb/src/data/executor/array_checkpoint.rs +++ b/nodedb/src/data/executor/array_checkpoint.rs @@ -7,7 +7,7 @@ //! `ArrayStore::memtable` is a plain in-memory `Memtable` (see //! `engine::array::memtable`). Every `INSERT INTO ARRAY` / `DELETE FROM ARRAY` //! lands there and advances the core watermark (`dispatch::array::mutate` -//! calls `note_write_lsn` with the Control Plane's `wal_lsn`). The periodic +//! calls `note_write` with the Control Plane's `wal_lsn`). The periodic //! checkpoint reports those writes as durable, and the manager then removes //! the `ArrayPut` / `ArrayDelete` records below that LSN. The checkpoint must //! therefore flush every memtable first. An explicit `NDARRAY_FLUSH` and the @@ -275,6 +275,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, })) .expect("push request"); diff --git a/nodedb/src/data/executor/cascade_journal_tests.rs b/nodedb/src/data/executor/cascade_journal_tests.rs index 64a6c4da7..ad2265130 100644 --- a/nodedb/src/data/executor/cascade_journal_tests.rs +++ b/nodedb/src/data/executor/cascade_journal_tests.rs @@ -133,6 +133,7 @@ fn request(plan: PhysicalPlan, wal_lsn: Option) -> Request { wal_lsn, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Admitted, } } diff --git a/nodedb/src/data/executor/columnar_checkpoint/load.rs b/nodedb/src/data/executor/columnar_checkpoint/load.rs index 907f57da6..5543baf6c 100644 --- a/nodedb/src/data/executor/columnar_checkpoint/load.rs +++ b/nodedb/src/data/executor/columnar_checkpoint/load.rs @@ -312,6 +312,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/columnar_checkpoint/write.rs b/nodedb/src/data/executor/columnar_checkpoint/write.rs index bdf89a45a..e08c17243 100644 --- a/nodedb/src/data/executor/columnar_checkpoint/write.rs +++ b/nodedb/src/data/executor/columnar_checkpoint/write.rs @@ -40,7 +40,7 @@ impl CoreLoop { /// applied before the export and replayed again after it duplicates the /// row. Every columnar record that mutates an engine therefore notes its /// LSN after applying: `execute_columnar_insert`, `execute_columnar_update` - /// and `execute_columnar_delete` call `note_collection_write_lsn`, which + /// and `execute_columnar_delete` call `note_collection_write`, which /// records the LSN as applied. /// /// A record whose live execution affected ZERO rows notes no LSN and so diff --git a/nodedb/src/data/executor/core_loop/event_emit_engines.rs b/nodedb/src/data/executor/core_loop/event_emit_engines.rs index 9f7697678..8cc1103c4 100644 --- a/nodedb/src/data/executor/core_loop/event_emit_engines.rs +++ b/nodedb/src/data/executor/core_loop/event_emit_engines.rs @@ -90,7 +90,7 @@ impl CoreLoop { /// The node-label write is already durable at its WAL LSN by the time this /// runs (the record was appended in the Control Plane before dispatch), so we /// advance the core watermark to it — exactly as every other write chokepoint - /// does via `note_write_lsn` — before emitting. That makes the forward + /// does via `note_write` — before emitting. That makes the forward /// event's LSN equal the WAL record's LSN the Event-Plane replay uses, /// satisfying watermark dedup. pub(in crate::data::executor) fn emit_graph_label_event( @@ -150,7 +150,7 @@ impl CoreLoop { /// surrogates from the plan. /// /// The caller MUST have advanced the core watermark to this edge's WAL LSN - /// (via `note_edge_write_lsn`) before calling, so the forward event's LSN + /// (via `note_edge_write`) before calling, so the forward event's LSN /// matches the WAL-replay reconstruction. pub(in crate::data::executor) fn emit_graph_edge_event( &mut self, diff --git a/nodedb/src/data/executor/core_loop/index_value_versions.rs b/nodedb/src/data/executor/core_loop/index_value_versions.rs index 023f7f616..e6bd0330e 100644 --- a/nodedb/src/data/executor/core_loop/index_value_versions.rs +++ b/nodedb/src/data/executor/core_loop/index_value_versions.rs @@ -3,19 +3,19 @@ //! Per-core, per-index write-VALUE version substrate. //! //! Sibling of [`super::write_index::WriteVersionIndex`]. That index records, -//! per written key/collection, the WAL LSN of the last committed write; this -//! one records — per `(database, tenant, collection, field)` secondary-index -//! dimension — the last committed-write LSN against each distinct indexed -//! VALUE. A later change validates an index-range read against the max LSN over -//! the read's value range instead of the coarse collection floor. +//! per written key/collection, the version of the last committed write; this +//! one records — per `(vshard, database, tenant, collection, field)` +//! secondary-index dimension — the last committed-write version against each +//! distinct indexed VALUE. An index-range read validates against the max +//! version over its value range instead of the coarse collection version. //! -//! Presence of the outer `(db, tenant, collection, field)` key is the -//! monotonic "tracked" flag: once any autocommit write records a value for a -//! field, that field stays tracked (horizon GC keeps the — now possibly empty -//! — inner map). An untracked field returns `None` (the validator falls back -//! to the collection floor); a tracked field with no in-range value returns -//! `Some(ZERO)`. A tracked field MUST have had every autocommit write's values -//! recorded — completeness is the recording chokepoints' contract. +//! Presence of the outer dimension key is the monotonic "tracked" flag: once +//! any committed write records a value for a field, that field stays tracked +//! (horizon GC keeps the, possibly empty, inner map). An untracked field +//! answers [`IndexVersion::Untracked`] (the validator falls back to the +//! collection version). A tracked field MUST have had every committed +//! write's values recorded: completeness is the recording chokepoints' +//! contract. //! //! Lives on the `!Send` core: plain `HashMap`/`BTreeMap`, no atomics/locks. @@ -23,59 +23,84 @@ use std::borrow::Borrow; use std::collections::{BTreeMap, HashMap}; use std::hash::{Hash, Hasher}; -use nodedb_types::{DatabaseId, TenantId}; +use nodedb_types::{DatabaseId, TenantId, WriteVersion}; -use crate::types::Lsn; +use crate::types::VShardId; use super::CoreLoop; - -/// Horizon retain window for per-value entries, in LSNs. Mirrors the per-key -/// index so both substrates age out together. -const RETAIN_WINDOW: u64 = 16_384; +use super::write_index::RETAIN_WINDOW; +use super::write_index::WriteStamp; /// Hard upper bound on total per-value entries across every tracked dimension. const MAX_INDEX_VALUE_ENTRIES: usize = 65_536; /// Per-index dimension key: one secondary-index `field` of a -/// `(database, tenant, collection)`. +/// `(database, tenant, collection)` on one vShard. #[derive(Debug, Clone, PartialEq, Eq)] pub struct IndexKey { + pub vshard: VShardId, pub db: DatabaseId, pub tenant: TenantId, pub collection: Box, pub field: Box, } +/// What the substrate knows about the values a read observed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IndexVersion { + /// No committed write recorded a value of the dimension. + Untracked, + /// The dimension is tracked. The highest version of the values the read + /// covers, `None` when none of them has a recorded version. + Tracked(Option), +} + /// The identity of an index dimension expressed as borrowable parts, so a /// `HashMap` keyed by owned [`IndexKey`] can be probed with borrowed `&str`s /// without allocating an owned key per lookup. trait IndexDim { - fn parts(&self) -> (DatabaseId, TenantId, &str, &str); + fn parts(&self) -> (VShardId, DatabaseId, TenantId, &str, &str); } impl IndexDim for IndexKey { - fn parts(&self) -> (DatabaseId, TenantId, &str, &str) { - (self.db, self.tenant, &self.collection, &self.field) + fn parts(&self) -> (VShardId, DatabaseId, TenantId, &str, &str) { + ( + self.vshard, + self.db, + self.tenant, + &self.collection, + &self.field, + ) } } -/// Borrowed probe into the per-index map — holds `&str`s, allocates nothing. -struct IndexDimRef<'a> { - db: DatabaseId, - tenant: TenantId, - collection: &'a str, - field: &'a str, +/// One index dimension, borrowed: the probe into the per-index map. Holds +/// `&str`s and allocates nothing. +#[derive(Debug, Clone, Copy)] +pub struct IndexDimRef<'a> { + pub vshard: VShardId, + pub db: DatabaseId, + pub tenant: TenantId, + pub collection: &'a str, + pub field: &'a str, } impl IndexDim for IndexDimRef<'_> { - fn parts(&self) -> (DatabaseId, TenantId, &str, &str) { - (self.db, self.tenant, self.collection, self.field) + fn parts(&self) -> (VShardId, DatabaseId, TenantId, &str, &str) { + ( + self.vshard, + self.db, + self.tenant, + self.collection, + self.field, + ) } } impl Hash for dyn IndexDim + '_ { fn hash(&self, state: &mut H) { - let (db, tenant, collection, field) = self.parts(); + let (vshard, db, tenant, collection, field) = self.parts(); + vshard.hash(state); db.hash(state); tenant.hash(state); collection.hash(state); @@ -106,75 +131,50 @@ impl Hash for IndexKey { /// Per-core, per-index write-VALUE version index. #[derive(Default)] pub struct IndexValueVersionIndex { - per_index: HashMap, Lsn>>, + per_index: HashMap, WriteVersion>>, } impl IndexValueVersionIndex { - /// Record a committed write of `value` on `(db, tenant, collection, field)` - /// at `lsn`. Marks the dimension tracked if new; advances the value slot to - /// `max(current, lsn)`. - pub fn record( - &mut self, - db: DatabaseId, - tenant: TenantId, - collection: &str, - field: &str, - value: &str, - lsn: Lsn, - ) { + /// Record a committed write of `value` on `dim` at `version`. Marks the + /// dimension tracked if new; advances the value slot to + /// `max(current, version)`. + pub fn record(&mut self, dim: IndexDimRef<'_>, value: &str, version: WriteVersion) { let key = IndexKey { - db, - tenant, - collection: Box::from(collection), - field: Box::from(field), + vshard: dim.vshard, + db: dim.db, + tenant: dim.tenant, + collection: Box::from(dim.collection), + field: Box::from(dim.field), }; let values = self.per_index.entry(key).or_default(); - let slot = values.entry(Box::from(value)).or_insert(Lsn::ZERO); - if lsn > *slot { - *slot = lsn; - } + let slot = values.entry(Box::from(value)).or_default(); + *slot = (*slot).max(version); } - /// Max committed-write LSN for an exact indexed value. `None` == the - /// `(collection, field)` dimension is UNtracked (validator falls back to the - /// collection floor). `Some(ZERO)` == tracked but this value has no recorded - /// write on this core (never invalidated → always current). - pub(in crate::data::executor) fn eq_max_lsn( + /// The recorded version of an exact indexed value. + pub(in crate::data::executor) fn eq_version( &self, - db: DatabaseId, - tenant: TenantId, - collection: &str, - field: &str, + dim: IndexDimRef<'_>, value: &str, - ) -> Option { - let values = self.per_index.get(&IndexDimRef { - db, - tenant, - collection, - field, - } as &dyn IndexDim)?; - Some(values.get(value).copied().unwrap_or(Lsn::ZERO)) + ) -> IndexVersion { + match self.dimension(dim) { + None => IndexVersion::Untracked, + Some(values) => IndexVersion::Tracked(values.get(value).copied()), + } } - /// Max committed-write LSN over the inclusive value range `[lo, hi]` (a - /// `None` bound is open). `None` == untracked dimension; `Some(ZERO)` == - /// tracked with no in-range recorded write. - pub(in crate::data::executor) fn range_max_lsn( + /// The highest recorded version over the inclusive value range `[lo, hi]` + /// (a `None` bound is open). + pub(in crate::data::executor) fn range_version( &self, - db: DatabaseId, - tenant: TenantId, - collection: &str, - field: &str, + dim: IndexDimRef<'_>, lo: Option<&str>, hi: Option<&str>, - ) -> Option { + ) -> IndexVersion { use std::ops::Bound; - let values = self.per_index.get(&IndexDimRef { - db, - tenant, - collection, - field, - } as &dyn IndexDim)?; + let Some(values) = self.dimension(dim) else { + return IndexVersion::Untracked; + }; let lo_b = match lo { Some(s) => Bound::Included(s), None => Bound::Unbounded, @@ -183,73 +183,91 @@ impl IndexValueVersionIndex { Some(s) => Bound::Included(s), None => Bound::Unbounded, }; - let mut max = Lsn::ZERO; - for (_v, lsn) in values.range::((lo_b, hi_b)) { - if *lsn > max { - max = *lsn; - } - } - Some(max) + IndexVersion::Tracked( + values + .range::((lo_b, hi_b)) + .map(|(_value, version)| *version) + .max(), + ) } - /// Test accessor: the recorded LSN for a `(db, tenant, collection, field)` - /// dimension's `value`, or `None` if the dimension or value is untracked. + fn dimension(&self, dim: IndexDimRef<'_>) -> Option<&BTreeMap, WriteVersion>> { + self.per_index.get(&dim as &dyn IndexDim) + } + + /// Test accessor: the recorded version of a dimension's `value`, or + /// `None` if the dimension or value is untracked. #[cfg(test)] - pub(in crate::data::executor) fn value_lsn( + pub(in crate::data::executor) fn value_version( &self, - db: DatabaseId, - tenant: TenantId, - collection: &str, - field: &str, + dim: IndexDimRef<'_>, value: &str, - ) -> Option { - self.per_index - .get(&IndexDimRef { - db, - tenant, - collection, - field, - } as &dyn IndexDim) + ) -> Option { + self.dimension(dim) .and_then(|values| values.get(value).copied()) } - /// Horizon GC against `watermark`, mirroring the per-key index. Evicts value - /// entries below `watermark - RETAIN_WINDOW`, keeping the (possibly empty) - /// inner map so the dimension stays tracked. A count backstop drops the - /// lowest-LSN entries under a TOTAL order so tied-LSN eviction is - /// replica-identical. - pub fn gc(&mut self, watermark: Lsn) { - let floor = watermark.as_u64().saturating_sub(RETAIN_WINDOW); + /// Forget every dimension of `vshard`: a snapshot install replaced its + /// rows. + pub(in crate::data::executor) fn drop_vshard(&mut self, vshard: VShardId) { + self.per_index.retain(|key, _| key.vshard != vshard); + } + + /// Horizon GC against each vShard's `latest` version, mirroring the + /// per-key index. Evicts value entries more than the retain window below + /// their vShard's latest version, keeping the (possibly empty) inner map + /// so the dimension stays tracked. A count backstop drops the lowest + /// versions under a TOTAL order so tied-version eviction is + /// replica-identical. Returns the highest version evicted per vShard. + pub fn gc( + &mut self, + latest: &HashMap, + ) -> HashMap { + let mut evicted: HashMap = HashMap::new(); let mut total = 0usize; - for values in self.per_index.values_mut() { - values.retain(|_, lsn| lsn.as_u64() >= floor); + for (key, values) in self.per_index.iter_mut() { + let latest = latest.get(&key.vshard).copied(); + values.retain(|_, version| { + let keep = latest + .and_then(|latest| latest.distance_above(*version)) + .is_none_or(|distance| distance <= RETAIN_WINDOW); + if !keep { + let slot = evicted.entry(key.vshard).or_default(); + *slot = (*slot).max(*version); + } + keep + }); total += values.len(); } if total > MAX_INDEX_VALUE_ENTRIES { let overflow = total - MAX_INDEX_VALUE_ENTRIES; - let mut all: Vec<(Lsn, IndexKey, Box)> = self + let mut all: Vec<(WriteVersion, IndexKey, Box)> = self .per_index .iter() .flat_map(|(k, values)| { values .iter() - .map(move |(v, lsn)| (*lsn, k.clone(), v.clone())) + .map(move |(v, version)| (*version, k.clone(), v.clone())) }) .collect(); all.sort_by(|a, b| { a.0.cmp(&b.0) + .then_with(|| a.1.vshard.cmp(&b.1.vshard)) .then_with(|| a.1.db.as_u64().cmp(&b.1.db.as_u64())) .then_with(|| a.1.tenant.as_u64().cmp(&b.1.tenant.as_u64())) .then_with(|| a.1.collection.cmp(&b.1.collection)) .then_with(|| a.1.field.cmp(&b.1.field)) .then_with(|| a.2.cmp(&b.2)) }); - for (_lsn, key, value) in all.into_iter().take(overflow) { + for (version, key, value) in all.into_iter().take(overflow) { + let slot = evicted.entry(key.vshard).or_default(); + *slot = (*slot).max(version); if let Some(values) = self.per_index.get_mut(&key) { values.remove(&value); } } } + evicted } } @@ -264,12 +282,21 @@ impl CoreLoop { tenant: TenantId, collection: &str, tuples: &[(String, String)], - lsn: Lsn, + stamp: WriteStamp, ) { + if tuples.is_empty() { + return; + } + let version = self.write_version_of(stamp); for (field, value) in tuples { - self.write_index - .index_values - .record(db, tenant, collection, field, value, lsn); + let dim = IndexDimRef { + vshard: stamp.vshard, + db, + tenant, + collection, + field, + }; + self.write_index.index_values.record(dim, value, version); } } } @@ -286,8 +313,17 @@ mod tests { TenantId::new(1) } + fn vshard() -> VShardId { + VShardId::new(3) + } + + fn at(index: u64) -> WriteVersion { + WriteVersion::logged(0, index) + } + fn key(field: &str) -> IndexKey { IndexKey { + vshard: vshard(), db: db(), tenant: tenant(), collection: Box::from("orders"), @@ -295,69 +331,72 @@ mod tests { } } + fn email() -> IndexDimRef<'static> { + IndexDimRef { + vshard: vshard(), + db: db(), + tenant: tenant(), + collection: "orders", + field: "email", + } + } + + fn record(index: &mut IndexValueVersionIndex, value: &str, version: WriteVersion) { + index.record(email(), value, version); + } + + fn latest(version: WriteVersion) -> HashMap { + HashMap::from([(vshard(), version)]) + } + #[test] - fn record_tracks_dimension_and_stores_value_lsn() { + fn record_tracks_dimension_and_stores_value_version() { let mut index = IndexValueVersionIndex::default(); - index.record(db(), tenant(), "orders", "email", "a@b.c", Lsn::new(10)); + record(&mut index, "a@b.c", at(10)); let values = index .per_index .get(&key("email")) .expect("dimension tracked after record"); - assert_eq!(values.get("a@b.c"), Some(&Lsn::new(10))); + assert_eq!(values.get("a@b.c"), Some(&at(10))); } #[test] fn record_is_monotonic_max_per_value() { let mut index = IndexValueVersionIndex::default(); - index.record(db(), tenant(), "orders", "email", "a@b.c", Lsn::new(10)); - index.record(db(), tenant(), "orders", "email", "a@b.c", Lsn::new(30)); - // A later record with a LOWER LSN must not regress the slot. - index.record(db(), tenant(), "orders", "email", "a@b.c", Lsn::new(20)); + record(&mut index, "a@b.c", at(10)); + record(&mut index, "a@b.c", at(30)); + record(&mut index, "a@b.c", at(20)); let values = index.per_index.get(&key("email")).expect("tracked"); - assert_eq!(values.get("a@b.c"), Some(&Lsn::new(30))); + assert_eq!(values.get("a@b.c"), Some(&at(30))); } #[test] - fn gc_horizon_evicts_but_keeps_dimension_tracked() { + fn another_vshard_does_not_track_the_dimension() { let mut index = IndexValueVersionIndex::default(); - index.record(db(), tenant(), "orders", "email", "a@b.c", Lsn::new(1)); - // Watermark far above the entry + retain window → entry evicted. - index.gc(Lsn::new(RETAIN_WINDOW + 100)); - let values = index - .per_index - .get(&key("email")) - .expect("dimension stays tracked after gc"); - assert!(values.is_empty(), "evicted value must not remain"); + record(&mut index, "a@b.c", at(10)); + let other = IndexDimRef { + vshard: VShardId::new(4), + ..email() + }; + assert_eq!(index.eq_version(other, "a@b.c"), IndexVersion::Untracked); } #[test] - fn gc_horizon_retains_entries_above_floor() { + fn gc_evicts_old_values_keeps_dimension_tracked_and_reports_the_bound() { let mut index = IndexValueVersionIndex::default(); - let watermark = Lsn::new(RETAIN_WINDOW + 100); - // Below floor (evicted) and above floor (retained). - index.record(db(), tenant(), "orders", "email", "old", Lsn::new(1)); - index.record( - db(), - tenant(), - "orders", - "email", - "new", - Lsn::new(watermark.as_u64()), - ); - index.gc(watermark); + record(&mut index, "old", at(1)); + record(&mut index, "new", at(RETAIN_WINDOW + 100)); + let evicted = index.gc(&latest(at(RETAIN_WINDOW + 100))); let values = index.per_index.get(&key("email")).expect("tracked"); assert_eq!(values.get("old"), None); - assert_eq!(values.get("new"), Some(&watermark)); + assert_eq!(values.get("new"), Some(&at(RETAIN_WINDOW + 100))); + assert_eq!(evicted.get(&vshard()), Some(&at(1))); } #[test] fn gc_count_backstop_is_insert_order_independent() { - // Overflow the count backstop by one, with all entries tied at the same - // LSN so eviction order falls entirely to the total order over - // (db, tenant, collection, field, value). The surviving set must be the - // same regardless of insertion order. let n = MAX_INDEX_VALUE_ENTRIES + 1; - let lsn = Lsn::new(1_000_000); + let version = at(1_000_000); let build = |ascending: bool| { let mut index = IndexValueVersionIndex::default(); @@ -366,12 +405,9 @@ mod tests { order.reverse(); } for i in order { - let value = format!("{i:08}"); - index.record(db(), tenant(), "orders", "email", &value, lsn); + record(&mut index, &format!("{i:08}"), version); } - // Watermark keeps every entry above the horizon floor, so only the - // count backstop fires. - index.gc(lsn); + index.gc(&latest(version)); index }; @@ -379,11 +415,10 @@ mod tests { let desc = build(false); assert_eq!(asc.per_index, desc.per_index); - // The lowest value under the total order is dropped, the next survives. let dropped = format!("{:08}", 0); let survivor = format!("{:08}", 1); let values = asc.per_index.get(&key("email")).expect("tracked"); assert_eq!(values.get(dropped.as_str()), None); - assert_eq!(values.get(survivor.as_str()), Some(&lsn)); + assert_eq!(values.get(survivor.as_str()), Some(&version)); } } diff --git a/nodedb/src/data/executor/core_loop/pressure/fixtures.rs b/nodedb/src/data/executor/core_loop/pressure/fixtures.rs index ef9da6963..4d61bfeda 100644 --- a/nodedb/src/data/executor/core_loop/pressure/fixtures.rs +++ b/nodedb/src/data/executor/core_loop/pressure/fixtures.rs @@ -91,6 +91,7 @@ pub(super) fn make_stub_request(id: u64) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), } } diff --git a/nodedb/src/data/executor/core_loop/priority_queues.rs b/nodedb/src/data/executor/core_loop/priority_queues.rs index e5abaf441..7febb77c9 100644 --- a/nodedb/src/data/executor/core_loop/priority_queues.rs +++ b/nodedb/src/data/executor/core_loop/priority_queues.rs @@ -359,6 +359,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/core_loop/response.rs b/nodedb/src/data/executor/core_loop/response.rs index cc16de630..05d2251ab 100644 --- a/nodedb/src/data/executor/core_loop/response.rs +++ b/nodedb/src/data/executor/core_loop/response.rs @@ -42,7 +42,7 @@ impl CoreLoop { partial: false, payload: Payload::empty(), watermark_lsn: self.watermark, - read_version_lsn: self.read_version_lsn(task), + read_versions: self.read_versions(task), error_code: None, stage_vote: None, write_set: Vec::new(), @@ -61,7 +61,7 @@ impl CoreLoop { partial: false, payload: Payload::from_vec(payload), watermark_lsn: self.watermark, - read_version_lsn: self.read_version_lsn(task), + read_versions: self.read_versions(task), error_code: None, stage_vote: None, write_set: Vec::new(), @@ -106,56 +106,110 @@ impl CoreLoop { partial: true, payload: Payload::from_vec(payload), watermark_lsn: self.watermark, - read_version_lsn: self.read_version_lsn(task), + read_versions: self.read_versions(task), error_code: None, stage_vote: None, write_set: Vec::new(), } } - /// Per-collection read-version LSN for `task`'s plan (distinct from the - /// core-global `watermark`). A graph read scoped to one collection reports - /// that collection's version: its edges are the collection's rows. On a write response this is the POST-write - /// version. A resolved KV write's version is the max across its - /// mutations, which may span two collections. - pub(in crate::data::executor) fn read_version_lsn( + /// The write versions `task` observed, one per vShard (distinct from the + /// core-global `watermark`). + /// + /// A write reports the POST-write version of each collection it wrote, on + /// its own vShard. A graph read reports every vShard's latest version on + /// this core: its edges live on their endpoints' vShards. An array read + /// reports the array's version on the vShard its task covers, held or + /// not. Any other read reports its + /// collection's version on the collection's home vShard, where the read + /// validates. A core reports only vShards it holds versions of. + pub(in crate::data::executor) fn read_versions( &self, task: &ExecutionTask, - ) -> crate::types::Lsn { + ) -> crate::types::ReadVersions { + let mut versions = crate::types::ReadVersions::new(); + let plan = task.plan(); + if task.wal_lsn().is_none() + && let crate::bridge::envelope::PhysicalPlan::Graph(_) = plan + { + for (vshard, latest) in self.write_index.latest_by_vshard() { + versions.note(vshard, latest); + } + return versions; + } + if task.wal_lsn().is_none() + && matches!( + plan, + crate::bridge::envelope::PhysicalPlan::Array(_) + | crate::bridge::envelope::PhysicalPlan::ClusterArray(_) + ) + && let Some(array) = plan.collection() + { + // The task's vShard is the one the read covers: the array's own + // vShard, or the shard a cluster slice leg reads. It is reported + // even when no write of it reached this core, at the bound its + // state holds, so a later first write of it makes the read stale. + let vshard = task.request.vshard_id; + versions.note( + vshard, + self.write_index.collection_current( + task.request.database_id, + task.request.tenant_id, + vshard, + array, + ), + ); + return versions; + } + let mut note = |collection: &str| { + let vshard = match task.wal_lsn() { + Some(_) => task.request.vshard_id, + None => match nodedb_types::CollectionKey::from_qualified_str( + task.request.database_id, + collection, + ) { + Ok(key) => key.vshard(), + Err(_) => return, + }, + }; + if self.write_index.holds(vshard) { + versions.note( + vshard, + self.write_index.collection_current( + task.request.database_id, + task.request.tenant_id, + vshard, + collection, + ), + ); + } + }; if let crate::bridge::envelope::PhysicalPlan::Kv( nodedb_physical::physical_plan::KvOp::ResolvedWrite { mutations, .. }, - ) = task.plan() + ) = plan { - return mutations - .iter() - .map(|m| self.collection_read_version(task, m.collection().as_str())) - .max() - .unwrap_or(crate::types::Lsn::ZERO); + for mutation in mutations { + note(mutation.collection().as_str()); + } + } else if let Some(collection) = plan.collection() { + note(collection); } - task.plan() - .collection() - .or_else(|| graph_read_collection(task.plan())) - .map(|c| self.collection_read_version(task, c)) - .unwrap_or(crate::types::Lsn::ZERO) - } - - /// One collection's recorded write LSN on this core, or `Lsn::ZERO` when - /// it has none. - fn collection_read_version(&self, task: &ExecutionTask, collection: &str) -> crate::types::Lsn { - self.write_index - .collection_write_lsn(&super::write_index::CollKey { - db: task.request.database_id, - tenant: task.request.tenant_id, - collection: Box::from(collection), - }) - .unwrap_or(crate::types::Lsn::ZERO) + versions } + /// An error answer to `task`. A `NotFound` answer is a read that found + /// no row, which a transaction validates, so it carries the versions the + /// read observed like any answered read. pub(in crate::data::executor) fn response_error( &self, task: &ExecutionTask, error_code: impl Into, ) -> Response { + let error_code = error_code.into(); + let read_versions = match error_code { + ErrorCode::NotFound => self.read_versions(task), + _ => crate::types::ReadVersions::new(), + }; Response { request_id: task.request_id(), status: Status::Error, @@ -163,8 +217,8 @@ impl CoreLoop { partial: false, payload: Payload::empty(), watermark_lsn: self.watermark, - read_version_lsn: crate::types::Lsn::ZERO, - error_code: Some(Box::new(error_code.into())), + read_versions, + error_code: Some(Box::new(error_code)), stage_vote: None, write_set: Vec::new(), } @@ -310,23 +364,6 @@ impl CoreLoop { } } -/// The collection a collection-scoped graph read walks. Graph plans report no -/// collection to routing (an edge homes on its endpoints), but a scoped read -/// still observes that one collection's edges. -fn graph_read_collection(plan: &crate::bridge::envelope::PhysicalPlan) -> Option<&str> { - use nodedb_physical::physical_plan::GraphOp; - match plan { - crate::bridge::envelope::PhysicalPlan::Graph( - GraphOp::Hop { collection, .. } - | GraphOp::Neighbors { collection, .. } - | GraphOp::NeighborsMulti { collection, .. } - | GraphOp::Path { collection, .. } - | GraphOp::Subgraph { collection, .. }, - ) => collection.as_ref().map(|c| c.as_str()), - _ => None, - } -} - #[cfg(test)] mod tests { use super::super::tests::{make_core_with_dir, make_default_task}; diff --git a/nodedb/src/data/executor/core_loop/state.rs b/nodedb/src/data/executor/core_loop/state.rs index fde56a685..1d3b37efd 100644 --- a/nodedb/src/data/executor/core_loop/state.rs +++ b/nodedb/src/data/executor/core_loop/state.rs @@ -478,8 +478,8 @@ pub struct CoreLoop { pub(in crate::data::executor) ts_resolve_holds: HashMap<(DatabaseId, TenantId, String), std::collections::HashSet>, - /// Per-core last-write-LSN version index (per key + per collection), - /// advanced by every committed write-apply. Type + GC in `write_index.rs`. + /// Per-core write-version index (per key + per collection, per vShard), + /// advanced by every committed write-apply. Type + GC in `write_index/`. pub(in crate::data::executor) write_index: super::write_index::WriteVersionIndex, /// The version keys the current record's apply forces. diff --git a/nodedb/src/data/executor/core_loop/tick.rs b/nodedb/src/data/executor/core_loop/tick.rs index 4c5739086..6a01bacad 100644 --- a/nodedb/src/data/executor/core_loop/tick.rs +++ b/nodedb/src/data/executor/core_loop/tick.rs @@ -98,7 +98,7 @@ impl CoreLoop { // The task never started, so nothing it would write ran. error_code: Some(Box::new(ErrorCode::ExpiredBeforeExecution)), stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } else { @@ -266,6 +266,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/src/data/executor/core_loop/write_index.rs b/nodedb/src/data/executor/core_loop/write_index.rs deleted file mode 100644 index 16bc04f08..000000000 --- a/nodedb/src/data/executor/core_loop/write_index.rs +++ /dev/null @@ -1,1371 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Per-core last-write-LSN version index. -//! -//! Records, for every committed write applied on this Data-Plane core, the WAL -//! LSN of the write against the written key (`last_write_lsn`) and against the -//! written collection (`coll_write_lsn`). The WAL LSN is allocated in the -//! Control Plane at wal-dispatch time and threaded onto the write task; the -//! apply chokepoints on this core feed it here. -//! -//! This is the shard-local write-version substrate the optimistic-concurrency -//! commit path validates a transaction's read-set against (see -//! [`CoreLoop::read_set_still_current`]). Because the index lives on the -//! `!Send` core it is a plain `HashMap`: no atomics, no locks, no cross-core -//! sharing. -//! -//! The per-key map is bounded: horizon GC (run from the periodic maintenance -//! hook) evicts entries far below the core watermark and enforces a hard -//! entry-count backstop. The per-collection map is bounded by the number of -//! live collections and is never LSN-GC'd. - -use std::collections::HashMap; - -use nodedb_types::calvin::{ReadKeyIdent, VersionedReadEntry}; -use nodedb_types::{DatabaseId, TenantId}; - -use crate::types::Lsn; - -use super::CoreLoop; - -/// Row identity type, re-exported from its plane-neutral home -/// ([`crate::types::KeyRepr`]) so Data-Plane call sites can keep referring to -/// it through this module. Read keys and write keys share this one namespace. -pub use crate::types::KeyRepr; - -/// Horizon retain window for per-key entries, in LSNs. Horizon GC evicts any -/// `last_write_lsn` entry whose LSN is more than this far below the core -/// watermark. Sized in the same order of magnitude as the idempotency-cache -/// cap (16,384 entries): a bounded recent-write history — enough to validate -/// in-flight transactions — not an unbounded write log. -const RETAIN_WINDOW: u64 = 16_384; - -/// Hard upper bound on the `last_write_lsn` entry count. When horizon GC leaves -/// more entries than this (a burst of distinct keys all inside the retain -/// window), the lowest-LSN (oldest) entries are dropped until the map is back -/// within bound. -const MAX_KEY_ENTRIES: usize = 65_536; - -/// Fully-qualified per-key version-index key. Scoped by `(database, tenant)` -/// exactly like the write path, so two tenants (or databases) never alias. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct WriteKey { - pub db: DatabaseId, - pub tenant: TenantId, - pub collection: Box, - pub key: KeyRepr, -} - -/// Fully-qualified per-collection version-index key. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CollKey { - pub db: DatabaseId, - pub tenant: TenantId, - pub collection: Box, -} - -/// Per-core last-write-LSN version index. -#[derive(Default)] -pub struct WriteVersionIndex { - /// Last committed-write LSN per written key. - last_write_lsn: HashMap, - /// Last committed-write LSN per written collection (the phantom-safe floor: - /// a predicate reader validates against this when it owns no per-key entry). - coll_write_lsn: HashMap, - /// Per-secondary-index-dimension write-VALUE versions — the finer-grained - /// sibling of `coll_write_lsn` an index-range read validates against. - pub(in crate::data::executor) index_values: super::index_value_versions::IndexValueVersionIndex, -} - -impl WriteVersionIndex { - pub fn new() -> Self { - Self::default() - } - - /// Record a committed write at `lsn`. - /// - /// Always advances the collection floor `coll_write_lsn[collection]` to the - /// max of its current value and `lsn`. When `key` is `Some`, also advances - /// the per-key version `last_write_lsn[key]` monotonically. Advancing the - /// core watermark is the caller's responsibility (see - /// [`CoreLoop::note_write_lsn`]). - pub fn note_write_lsn( - &mut self, - db: DatabaseId, - tenant: TenantId, - collection: &str, - key: Option, - lsn: Lsn, - ) { - let coll_key = CollKey { - db, - tenant, - collection: Box::from(collection), - }; - let slot = self.coll_write_lsn.entry(coll_key).or_insert(Lsn::ZERO); - if lsn > *slot { - *slot = lsn; - } - - if let Some(key) = key { - let write_key = WriteKey { - db, - tenant, - collection: Box::from(collection), - key, - }; - let slot = self.last_write_lsn.entry(write_key).or_insert(Lsn::ZERO); - if lsn > *slot { - *slot = lsn; - } - } - } - - /// Current per-key version, if recorded. - pub(crate) fn key_write_lsn(&self, key: &WriteKey) -> Option { - self.last_write_lsn.get(key).copied() - } - - /// Current per-collection floor version, if recorded. - pub(crate) fn collection_write_lsn(&self, key: &CollKey) -> Option { - self.coll_write_lsn.get(key).copied() - } - - /// Every recorded per-key version. - #[cfg(test)] - pub(crate) fn recorded_keys(&self) -> impl Iterator { - self.last_write_lsn.iter().map(|(key, lsn)| (key, *lsn)) - } - - /// Every recorded per-collection floor. - #[cfg(test)] - pub(crate) fn recorded_collections(&self) -> impl Iterator { - self.coll_write_lsn.iter().map(|(key, lsn)| (key, *lsn)) - } - - /// Whether a previously observed read is still current against this - /// core's recorded write versions. - /// - /// A `Point` read is current iff no write to that exact key has been - /// recorded since the read (`last_write_lsn <= read_lsn`); a key with no - /// recorded write has never been written on this core since the read, so - /// it is treated as version zero and is always current. A `Predicate` - /// read is current iff no write to the collection has been recorded - /// since the read, checked the same way against the collection floor. - pub(crate) fn read_is_valid( - &self, - db: DatabaseId, - tenant: TenantId, - collection: &str, - key: &ReadKeyIdent, - read_lsn: Lsn, - ) -> bool { - // Collection-floor check shared by `Predicate` and the fallback for - // untracked secondary-index dimensions: a read is current iff no write - // to the collection has been recorded since it. `IndexEq` / `IndexRange` - // consult the per-value substrate first and only fall back here when the - // `(collection, field)` dimension is untracked. - let collection_floor_current = || { - let coll_key = CollKey { - db, - tenant, - collection: Box::from(collection), - }; - self.collection_write_lsn(&coll_key).unwrap_or(Lsn::ZERO) <= read_lsn - }; - - match key { - ReadKeyIdent::Point(repr) => { - let write_key = WriteKey { - db, - tenant, - collection: Box::from(collection), - key: repr.clone(), - }; - self.key_write_lsn(&write_key).unwrap_or(Lsn::ZERO) <= read_lsn - } - ReadKeyIdent::Predicate => collection_floor_current(), - ReadKeyIdent::IndexEq { field, value } => { - match self - .index_values - .eq_max_lsn(db, tenant, collection, field, value) - { - Some(max) => max <= read_lsn, - None => collection_floor_current(), - } - } - ReadKeyIdent::IndexRange { field, lo, hi } => { - match self.index_values.range_max_lsn( - db, - tenant, - collection, - field, - lo.as_deref(), - hi.as_deref(), - ) { - Some(max) => max <= read_lsn, - None => collection_floor_current(), - } - } - } - } - - /// Horizon garbage-collect the per-key map against `watermark`. - /// - /// Evicts every entry whose LSN falls below `watermark - RETAIN_WINDOW`, - /// then, if more than [`MAX_KEY_ENTRIES`] remain, drops the lowest-LSN - /// entries until back within bound. The per-collection map is bounded by - /// the live-collection count and is intentionally left untouched. - pub fn gc(&mut self, watermark: Lsn) { - let floor = watermark.as_u64().saturating_sub(RETAIN_WINDOW); - self.last_write_lsn.retain(|_, lsn| lsn.as_u64() >= floor); - - if self.last_write_lsn.len() > MAX_KEY_ENTRIES { - let overflow = self.last_write_lsn.len() - MAX_KEY_ENTRIES; - // Drop the `overflow` oldest (lowest-LSN) entries. - let mut by_lsn: Vec<(Lsn, WriteKey)> = self - .last_write_lsn - .iter() - .map(|(k, lsn)| (*lsn, k.clone())) - .collect(); - // TOTAL order so tied-LSN eviction is replica-identical: a plain - // `sort_by_key(lsn)` over a `HashMap`-collected Vec would let the - // dropped set depend on hash-iteration layout, diverging across - // replicas. `DatabaseId`/`TenantId` lack `Ord` (compare via - // `as_u64()`); `KeyRepr` is `Ord`. - by_lsn.sort_by(|a, b| { - a.0.cmp(&b.0) - .then_with(|| a.1.db.as_u64().cmp(&b.1.db.as_u64())) - .then_with(|| a.1.tenant.as_u64().cmp(&b.1.tenant.as_u64())) - .then_with(|| a.1.collection.cmp(&b.1.collection)) - .then_with(|| a.1.key.cmp(&b.1.key)) - }); - for (_, key) in by_lsn.into_iter().take(overflow) { - self.last_write_lsn.remove(&key); - } - } - - self.index_values.gc(watermark); - } -} - -impl CoreLoop { - /// Record a committed write into the per-core version index and advance the - /// core watermark monotonically. - /// - /// Called once per written key at every Data-Plane apply chokepoint, using - /// the WAL LSN the Control Plane allocated at wal-dispatch and threaded onto - /// the write task. `key` is `None` for engines whose per-key identity is - /// internal (columnar / timeseries / array / spatial / FTS) — those record - /// only the collection floor. - /// - /// In the install pass of a committed-redo apply the version waits in the - /// apply scope. The apply publishes it once the record settled, so a - /// rolled-back install leaves no version and no watermark behind. - pub(in crate::data::executor) fn note_write_lsn( - &mut self, - db: DatabaseId, - tenant: TenantId, - collection: &str, - key: Option, - lsn: Lsn, - ) { - if let Some(scope) = self.redo_apply.scope.as_mut() { - scope.defer_write_version(db, tenant, collection, key, lsn); - return; - } - self.publish_write_version(db, tenant, collection, key, lsn); - } - - /// Record a write version and advance the core watermark monotonically. - /// - /// Every applied write reaches here, so this is also where the core notes - /// the write's LSN as applied: a checkpoint's replay stamp names it. - pub(in crate::data::executor) fn publish_write_version( - &mut self, - db: DatabaseId, - tenant: TenantId, - collection: &str, - key: Option, - lsn: Lsn, - ) { - self.floors.applied_prefix.note_applied(lsn); - self.write_index - .note_write_lsn(db, tenant, collection, key, lsn); - if lsn > self.watermark { - self.watermark = lsn; - } - } - - /// Record a committed write's collection floor only (no per-key entry), - /// if a WAL LSN was threaded onto `task`. Shared by the columnar-family - /// write handlers (columnar / timeseries / array / spatial / FTS) whose - /// per-key identity is internal — a predicate reader validates against the - /// collection floor when it owns no per-key version. - pub(in crate::data::executor) fn note_collection_write_lsn( - &mut self, - task: &super::super::task::ExecutionTask, - collection: &str, - ) { - if let Some(lsn) = task.wal_lsn() { - self.note_write_lsn( - task.request.database_id, - task.request.tenant_id, - collection, - None, - lsn, - ); - } - } - - /// Run horizon GC on the per-core version index. Invoked from the periodic - /// maintenance hook — no dedicated timer. - pub(in crate::data::executor) fn gc_write_index(&mut self) { - self.write_index.gc(self.watermark); - } - - /// Whether this shard's slice of a transaction's LSN-versioned read-set was - /// still current against the local write versions. - /// - /// Filters the read-set to the entries homed on this request's vShard — - /// the only reads this core holds versions for — then checks each against - /// the per-core write-version index via - /// [`WriteVersionIndex::read_is_valid`]. Short-circuits on the first entry - /// that is no longer current. An empty or fully-remote slice is vacuously - /// current (`true`). The `(database, tenant)` scope mirrors the write-version - /// recorder so a read validates against the same key space it was recorded - /// in. - /// - /// An entry's home is its `home_vshard` when set (a cross-shard graph read - /// names the key vShard it read edges on), else its collection's vShard, - /// by the same collection-in-database function the scheduler routes plans - /// with. A homed entry with no collection observed every collection on its - /// vShard, so it is current only while the core watermark has not passed - /// its read version. - pub(in crate::data::executor) fn read_set_still_current( - &self, - task: &super::super::task::ExecutionTask, - tid: u64, - versioned_reads: &[VersionedReadEntry], - ) -> bool { - let db = task.request.database_id; - let tenant = TenantId::new(tid); - let local_vshard = task.request.vshard_id.as_u32(); - versioned_reads.iter().all(|entry| { - if let Some(home) = entry.home_vshard { - if home != local_vshard { - return true; - } - if entry.collection.is_empty() { - return self.watermark <= entry.read_lsn; - } - return self.write_index.read_is_valid( - db, - tenant, - &entry.collection, - &entry.key, - entry.read_lsn, - ); - } - // An entry carries the plan's database-qualified name. One that does - // not de-qualify cannot be homed or validated, so the read set fails - // closed. - match nodedb_types::CollectionKey::from_qualified_str(db, &entry.collection) { - Err(_) => false, - Ok(key) if key.vshard().as_u32() != local_vshard => true, - Ok(_) => self.write_index.read_is_valid( - db, - tenant, - &entry.collection, - &entry.key, - entry.read_lsn, - ), - } - }) - } - - /// Record a committed document/vector write's version, keyed by the - /// written row's cross-engine surrogate, if a WAL LSN was threaded onto - /// `task`. Shared by every per-surrogate write chokepoint (point put, - /// point insert, point delete, bulk update, bulk delete). - pub(in crate::data::executor) fn note_surrogate_write_lsn( - &mut self, - task: &super::super::task::ExecutionTask, - tid: u64, - collection: &str, - surrogate: u32, - ) { - if let Some(lsn) = task.wal_lsn() { - self.note_write_lsn( - task.request.database_id, - TenantId::new(tid), - collection, - Some(KeyRepr::Surrogate(surrogate)), - lsn, - ); - } - } - - /// Record a committed WAL-replay write's version. A no-op when - /// `record_lsn == 0` (no durable LSN was recorded for this write); `key` - /// is `None` for collection-only entries (e.g. truncate) and `Some` for - /// per-key/per-surrogate entries, exactly like [`Self::note_write_lsn`]. - /// - /// Shared by every WAL replay chokepoint (KV, document, document-vector): - /// unlike the live write path, replay only has the raw `(database_id, - /// tenant_id, record_lsn)` off the WAL record header, not an - /// `ExecutionTask` — hence the separate `u64`-typed entry point rather - /// than reusing [`Self::note_surrogate_write_lsn`] / - /// [`Self::note_collection_write_lsn`]. - pub(in crate::data::executor) fn note_replay_write_lsn( - &mut self, - database_id: u64, - tenant_id: u64, - collection: &str, - key: Option, - record_lsn: u64, - ) { - if record_lsn != 0 { - self.note_write_lsn( - DatabaseId::new(database_id), - TenantId::new(tenant_id), - collection, - key, - Lsn::new(record_lsn), - ); - } - } -} - -#[cfg(test)] -pub(crate) mod tests { - use std::time::{Duration, Instant}; - - use nodedb_bridge::buffer::RingBuffer; - use nodedb_physical::physical_plan::DocumentOp; - use nodedb_types::calvin::EngineTag; - use nodedb_types::{QualifiedCollection, Surrogate}; - - use super::*; - use crate::bridge::dispatch::{BridgeRequest, BridgeResponse}; - use crate::bridge::envelope::{PhysicalPlan, Priority, Request, Status}; - use crate::data::executor::core_loop::CoreLoop; - use crate::data::executor::handlers::graph::EdgePutParams; - use crate::data::executor::handlers::kv::crud::KvWriteParams; - use crate::data::executor::handlers::point::put::PointPutExec; - use crate::data::executor::task::ExecutionTask; - use crate::types::{ReadConsistency, RequestId, TraceId, VShardId}; - - fn db() -> DatabaseId { - DatabaseId::DEFAULT - } - - fn tenant() -> TenantId { - TenantId::new(1) - } - - #[test] - fn point_read_is_valid_when_key_never_written() { - let index = WriteVersionIndex::new(); - let key = ReadKeyIdent::Point(KeyRepr::Surrogate(7)); - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn point_read_is_valid_when_write_at_or_before_read_lsn() { - let mut index = WriteVersionIndex::new(); - index.note_write_lsn( - db(), - tenant(), - "orders", - Some(KeyRepr::Surrogate(7)), - Lsn::new(10), - ); - - let key = ReadKeyIdent::Point(KeyRepr::Surrogate(7)); - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(20))); - } - - #[test] - fn point_read_is_invalid_when_write_after_read_lsn() { - let mut index = WriteVersionIndex::new(); - index.note_write_lsn( - db(), - tenant(), - "orders", - Some(KeyRepr::Surrogate(7)), - Lsn::new(20), - ); - - let key = ReadKeyIdent::Point(KeyRepr::Surrogate(7)); - assert!(!index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn predicate_read_is_valid_when_collection_never_written() { - let index = WriteVersionIndex::new(); - let key = ReadKeyIdent::Predicate; - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn predicate_read_is_valid_when_floor_at_or_before_read_lsn() { - let mut index = WriteVersionIndex::new(); - index.note_write_lsn(db(), tenant(), "orders", None, Lsn::new(10)); - - let key = ReadKeyIdent::Predicate; - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(20))); - } - - #[test] - fn predicate_read_is_invalid_when_floor_after_read_lsn() { - let mut index = WriteVersionIndex::new(); - index.note_write_lsn(db(), tenant(), "orders", None, Lsn::new(20)); - - let key = ReadKeyIdent::Predicate; - assert!(!index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn untracked_index_dimension_falls_back_to_collection_floor() { - // A `(collection, field)` never recorded in the per-value substrate is - // untracked → `eq_max_lsn`/`range_max_lsn` return `None` → the validator - // falls back to the collection floor, producing the SAME verdict as a - // `Predicate` read for every floor position. - let index_eq = ReadKeyIdent::IndexEq { - field: "email".to_string(), - value: "a@b.c".to_string(), - }; - let index_range = ReadKeyIdent::IndexRange { - field: "age".to_string(), - lo: Some("18".to_string()), - hi: None, - }; - let predicate = ReadKeyIdent::Predicate; - - // Floor below the read LSN (current), at it (current), and above it - // (stale) — the untracked index variants track `Predicate` in every case. - for (floor, read_lsn) in [(5u64, 10u64), (10, 10), (20, 10)] { - let mut index = WriteVersionIndex::new(); - index.note_write_lsn(db(), tenant(), "orders", None, Lsn::new(floor)); - let want = - index.read_is_valid(db(), tenant(), "orders", &predicate, Lsn::new(read_lsn)); - assert_eq!( - index.read_is_valid(db(), tenant(), "orders", &index_eq, Lsn::new(read_lsn)), - want, - "untracked IndexEq must match Predicate (floor {floor}, read {read_lsn})" - ); - assert_eq!( - index.read_is_valid(db(), tenant(), "orders", &index_range, Lsn::new(read_lsn)), - want, - "untracked IndexRange must match Predicate (floor {floor}, read {read_lsn})" - ); - } - } - - #[test] - fn index_eq_disjoint_write_does_not_abort() { - // Read of email = "a@b.c" at read_lsn 10; a later write to a DIFFERENT - // value on the same dimension must NOT abort the read (the coarse - // collection floor would have). - let mut index = WriteVersionIndex::new(); - // A real write to a disjoint value advances BOTH the collection floor - // (which the coarse `Predicate` path would abort on) AND records the - // per-value entry — so this proves the per-value check reduces the abort, - // not merely that nothing was written. - index.note_write_lsn(db(), tenant(), "orders", None, Lsn::new(20)); - index - .index_values - .record(db(), tenant(), "orders", "email", "z@z.z", Lsn::new(20)); - let key = ReadKeyIdent::IndexEq { - field: "email".to_string(), - value: "a@b.c".to_string(), - }; - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn index_range_disjoint_write_does_not_abort() { - // Range [10, 20] read at read_lsn 10; a write to out-of-range "50" must - // not abort. - let mut index = WriteVersionIndex::new(); - // Advance the collection floor too (the coarse path aborts on it) so this - // proves range validation reduces the abort, not just an empty index. - index.note_write_lsn(db(), tenant(), "orders", None, Lsn::new(20)); - index - .index_values - .record(db(), tenant(), "orders", "age", "50", Lsn::new(20)); - let key = ReadKeyIdent::IndexRange { - field: "age".to_string(), - lo: Some("10".to_string()), - hi: Some("20".to_string()), - }; - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn index_eq_same_value_conflict_aborts() { - // A write to the SAME read value after the read LSN must abort. - let mut index = WriteVersionIndex::new(); - index - .index_values - .record(db(), tenant(), "orders", "email", "a@b.c", Lsn::new(20)); - let key = ReadKeyIdent::IndexEq { - field: "email".to_string(), - value: "a@b.c".to_string(), - }; - assert!(!index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn index_range_in_range_added_value_conflict_aborts() { - // An added value INSIDE the read range after the read LSN must abort. - let mut index = WriteVersionIndex::new(); - index - .index_values - .record(db(), tenant(), "orders", "age", "15", Lsn::new(20)); - let key = ReadKeyIdent::IndexRange { - field: "age".to_string(), - lo: Some("10".to_string()), - hi: Some("20".to_string()), - }; - assert!(!index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn index_range_in_range_removed_value_conflict_aborts() { - // A delete of an in-range value also records that value's LSN, so a - // removal inside the read range must abort the read just like an insert. - let mut index = WriteVersionIndex::new(); - index - .index_values - .record(db(), tenant(), "orders", "age", "17", Lsn::new(20)); - let key = ReadKeyIdent::IndexRange { - field: "age".to_string(), - lo: Some("10".to_string()), - hi: Some("20".to_string()), - }; - assert!(!index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn index_range_phantom_insert_aborts() { - // Phantom protection: the range is current while it holds no in-range - // value (tracked → `Some(ZERO)`), then a NEW in-range value recorded - // after the read LSN invalidates it — proving the range captures the - // predicate, not just values extant at read time. - let mut index = WriteVersionIndex::new(); - // Track the dimension with an OUT-of-range value so the read starts - // current (tracked, no in-range write). - index - .index_values - .record(db(), tenant(), "orders", "age", "99", Lsn::new(5)); - let key = ReadKeyIdent::IndexRange { - field: "age".to_string(), - lo: Some("10".to_string()), - hi: Some("20".to_string()), - }; - assert!( - index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10)), - "range with no in-range write is current" - ); - - // Phantom insert inside the range after the read LSN. - index - .index_values - .record(db(), tenant(), "orders", "age", "15", Lsn::new(20)); - assert!( - !index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10)), - "phantom in-range insert must abort" - ); - } - - #[test] - fn tracked_index_eq_missing_value_is_current() { - // A tracked dimension (some other value recorded) queried for a value - // with no entry returns `Some(ZERO)` → current. - let mut index = WriteVersionIndex::new(); - index - .index_values - .record(db(), tenant(), "orders", "email", "other@x.y", Lsn::new(20)); - let key = ReadKeyIdent::IndexEq { - field: "email".to_string(), - value: "a@b.c".to_string(), - }; - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - #[test] - fn tracked_index_range_empty_range_is_current() { - // A tracked dimension queried for a range with no in-range entry returns - // `Some(ZERO)` → current. - let mut index = WriteVersionIndex::new(); - index - .index_values - .record(db(), tenant(), "orders", "age", "99", Lsn::new(20)); - let key = ReadKeyIdent::IndexRange { - field: "age".to_string(), - lo: Some("10".to_string()), - hi: Some("20".to_string()), - }; - assert!(index.read_is_valid(db(), tenant(), "orders", &key, Lsn::new(10))); - } - - // ── CoreLoop-level fixtures shared with the tests below and with other - // executor test modules (`make_core_with_dir` / `make_default_task` are - // `pub` for exactly that reason). ──────────────────────────────────────── - - fn make_core() -> ( - CoreLoop, - nodedb_bridge::buffer::Producer, - nodedb_bridge::buffer::Consumer, - tempfile::TempDir, - ) { - let dir = tempfile::tempdir().unwrap(); - let (req_tx, req_rx) = RingBuffer::channel::(64); - let (resp_tx, resp_rx) = RingBuffer::channel::(64); - let core = CoreLoop::open( - 0, - req_rx, - resp_tx, - dir.path(), - std::sync::Arc::new(nodedb_types::OrdinalClock::new()), - crate::data::executor::core_loop::test_governor(), - ) - .unwrap(); - (core, req_tx, resp_rx, dir) - } - - pub fn make_core_with_dir( - dir: &std::path::Path, - ) -> ( - CoreLoop, - nodedb_bridge::buffer::Producer, - nodedb_bridge::buffer::Consumer, - ) { - let (req_tx, req_rx) = RingBuffer::channel::(64); - let (resp_tx, resp_rx) = RingBuffer::channel::(64); - let core = CoreLoop::open( - 0, - req_rx, - resp_tx, - dir, - std::sync::Arc::new(nodedb_types::OrdinalClock::new()), - crate::data::executor::core_loop::test_governor(), - ) - .unwrap(); - (core, req_tx, resp_rx) - } - - fn make_request(plan: PhysicalPlan) -> Request { - Request { - request_id: RequestId::new(1), - tenant_id: TenantId::new(1), - database_id: DatabaseId::DEFAULT, - vshard_id: VShardId::new(0), - plan, - deadline: Instant::now() + Duration::from_secs(5), - priority: Priority::Normal, - trace_id: TraceId::ZERO, - consistency: ReadConsistency::Strong, - idempotency_key: None, - event_source: crate::event::EventSource::User, - user_roles: Vec::new(), - user_id: None, - statement_digest: None, - txn_id: None, - wal_lsn: None, - resolved_now_ms: None, - commit_hlc: None, - admission: crate::bridge::envelope::Admission::Admitted, - } - } - - /// A minimal `ExecutionTask` (DEFAULT database/tenant, vShard 0, no WAL LSN) - /// for handler unit tests that only read `request.database_id`. The carried - /// plan is inert — edge/point handlers take their parameters directly. - pub fn make_default_task() -> crate::data::executor::task::ExecutionTask { - crate::data::executor::task::ExecutionTask::new(make_request(PhysicalPlan::Document( - DocumentOp::PointGet { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "x"), - document_id: "y".into(), - surrogate: None, - pk_bytes: Vec::new(), - rls_filters: Vec::new(), - system_time: nodedb_types::SystemTimeScope::Current, - valid_at_ms: None, - }, - ))) - } - - /// A `{k: v}` document body: a standard MessagePack map. - fn doc_value(k: &str, v: &str) -> Vec { - let mut obj = std::collections::HashMap::new(); - obj.insert(k.to_string(), nodedb_types::Value::String(v.into())); - nodedb_types::value_to_msgpack(&nodedb_types::Value::Object(obj)).unwrap() - } - - /// An `ExecutionTask` carrying a known WAL LSN, tenant 1 / database DEFAULT. - fn wal_task(lsn: u64) -> ExecutionTask { - let plan = PhysicalPlan::Document(DocumentOp::PointGet { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "x"), - document_id: "y".into(), - surrogate: None, - pk_bytes: Vec::new(), - rls_filters: Vec::new(), - system_time: nodedb_types::SystemTimeScope::Current, - valid_at_ms: None, - }); - ExecutionTask::with_wal_lsn(make_request(plan), Some(Lsn::new(lsn))) - } - - fn surrogate_key(collection: &str, surrogate: u32) -> WriteKey { - WriteKey { - db: DatabaseId::DEFAULT, - tenant: TenantId::new(1), - collection: Box::from(collection), - key: KeyRepr::Surrogate(surrogate), - } - } - - fn coll_key(collection: &str) -> CollKey { - CollKey { - db: DatabaseId::DEFAULT, - tenant: TenantId::new(1), - collection: Box::from(collection), - } - } - - #[test] - fn point_put_records_write_version_and_advances_watermark() { - let (mut core, _, _, _dir) = make_core(); - - let task = wal_task(10); - let resp = core.execute_point_put( - &task, - PointPutExec { - tid: 1, - collection: "orders", - document_id: "o1", - surrogate: Surrogate::new(7), - value: &doc_value("a", "1"), - returning: None, - rls_filters: &[], - resolved_sum_targets: &[], - }, - ); - assert_eq!(resp.status, Status::Ok); - - assert_eq!( - core.write_index.key_write_lsn(&surrogate_key("orders", 7)), - Some(Lsn::new(10)) - ); - assert_eq!( - core.write_index.collection_write_lsn(&coll_key("orders")), - Some(Lsn::new(10)) - ); - assert_eq!(core.watermark, Lsn::new(10)); - - // Second write to the same key with a larger LSN overwrites monotonically. - let task = wal_task(20); - core.execute_point_put( - &task, - PointPutExec { - tid: 1, - collection: "orders", - document_id: "o1", - surrogate: Surrogate::new(7), - value: &doc_value("a", "2"), - returning: None, - rls_filters: &[], - resolved_sum_targets: &[], - }, - ); - assert_eq!( - core.write_index.key_write_lsn(&surrogate_key("orders", 7)), - Some(Lsn::new(20)) - ); - assert_eq!( - core.write_index.collection_write_lsn(&coll_key("orders")), - Some(Lsn::new(20)) - ); - assert_eq!(core.watermark, Lsn::new(20)); - - // A lower LSN never regresses an existing entry or the watermark. - let task = wal_task(15); - core.execute_point_put( - &task, - PointPutExec { - tid: 1, - collection: "orders", - document_id: "o1", - surrogate: Surrogate::new(7), - value: &doc_value("a", "3"), - returning: None, - rls_filters: &[], - resolved_sum_targets: &[], - }, - ); - assert_eq!( - core.write_index.key_write_lsn(&surrogate_key("orders", 7)), - Some(Lsn::new(20)) - ); - assert_eq!(core.watermark, Lsn::new(20)); - - // A second collection tracks its own max independently. - let task = wal_task(30); - core.execute_point_put( - &task, - PointPutExec { - tid: 1, - collection: "items", - document_id: "i1", - surrogate: Surrogate::new(9), - value: &doc_value("a", "4"), - returning: None, - rls_filters: &[], - resolved_sum_targets: &[], - }, - ); - assert_eq!( - core.write_index.collection_write_lsn(&coll_key("items")), - Some(Lsn::new(30)) - ); - assert_eq!( - core.write_index.collection_write_lsn(&coll_key("orders")), - Some(Lsn::new(20)) - ); - assert_eq!(core.watermark, Lsn::new(30)); - } - - #[test] - fn kv_put_records_kvkey_version() { - let (mut core, _, _, _dir) = make_core(); - let task = wal_task(42); - let resp = core.execute_kv_put( - &task, - KvWriteParams { - did: DatabaseId::DEFAULT.as_u64(), - tid: 1, - collection: "kv", - key: b"k1".as_slice(), - value: b"v1".as_slice(), - ttl_ms: 0, - surrogate: Surrogate::new(3), - returning: None, - rls_filters: &[], - }, - ); - assert_eq!(resp.status, Status::Ok); - - let wk = WriteKey { - db: DatabaseId::DEFAULT, - tenant: TenantId::new(1), - collection: Box::from("kv"), - key: KeyRepr::KvKey(Box::from(b"k1".as_slice())), - }; - assert_eq!(core.write_index.key_write_lsn(&wk), Some(Lsn::new(42))); - assert_eq!( - core.write_index.collection_write_lsn(&coll_key("kv")), - Some(Lsn::new(42)) - ); - assert_eq!(core.watermark, Lsn::new(42)); - } - - #[test] - fn an_autocommit_apply_is_named_by_the_next_replay_stamp() { - let (mut core, _, _, _dir) = make_core(); - core.floors - .applied_prefix - .observe_outcome_floor(Lsn::new(40)); - let resp = core.execute_kv_put( - &wal_task(42), - KvWriteParams { - did: DatabaseId::DEFAULT.as_u64(), - tid: 1, - collection: "kv", - key: b"k1".as_slice(), - value: b"v1".as_slice(), - ttl_ms: 0, - surrogate: Surrogate::new(3), - returning: None, - rls_filters: &[], - }, - ); - assert_eq!(resp.status, Status::Ok); - - let stamp = core.floors.applied_prefix.stamp().expect("exact stamp"); - assert!( - stamp.skips(42), - "the applied write is in every later artifact" - ); - assert!( - !stamp.skips(41), - "a record between the floor and the applied write still replays" - ); - } - - #[test] - fn edge_put_records_edge_version() { - let (mut core, _, _, _dir) = make_core(); - let task = wal_task(50); - let resp = core.execute_edge_put( - &task, - EdgePutParams { - tid: 1, - collection: "graph", - src_id: "a", - label: "KNOWS", - dst_id: "b", - properties: &[], - src_surrogate: Surrogate::new(1), - dst_surrogate: Surrogate::new(2), - }, - ); - assert_eq!(resp.status, Status::Ok); - - let wk = WriteKey { - db: DatabaseId::DEFAULT, - tenant: TenantId::new(1), - collection: Box::from("graph"), - key: KeyRepr::Edge { - src: Box::from("a"), - label: Box::from("KNOWS"), - dst: Box::from("b"), - }, - }; - assert_eq!(core.write_index.key_write_lsn(&wk), Some(Lsn::new(50))); - assert_eq!(core.watermark, Lsn::new(50)); - } - - #[test] - fn a_committed_transaction_records_its_write_versions() { - let (mut core, _, _, _dir) = make_core(); - let task = wal_task(60); - let plans = vec![PhysicalPlan::Document(DocumentOp::PointPut { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "batch"), - document_id: "d1".into(), - value: doc_value("a", "1"), - surrogate: Surrogate::new(11), - pk_bytes: Vec::new(), - returning: None, - rls_filters: Vec::new(), - resolved_sum_targets: Vec::new(), - })]; - let resp = core.commit_plans_for_test(&task, 1, &plans, 60); - assert_eq!(resp.status, Status::Ok, "{:?}", resp.error_code); - - assert_eq!( - core.write_index.key_write_lsn(&surrogate_key("batch", 11)), - Some(Lsn::new(60)) - ); - assert_eq!( - core.write_index.collection_write_lsn(&coll_key("batch")), - Some(Lsn::new(60)) - ); - assert_eq!(core.watermark, Lsn::new(60)); - } - - #[test] - fn no_wal_lsn_records_nothing() { - let (mut core, _, _, _dir) = make_core(); - // Task without a WAL LSN — the version index is skipped, not advanced. - let task = ExecutionTask::new(make_request(PhysicalPlan::Document(DocumentOp::PointGet { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "x"), - document_id: "y".into(), - surrogate: None, - pk_bytes: Vec::new(), - rls_filters: Vec::new(), - system_time: nodedb_types::SystemTimeScope::Current, - valid_at_ms: None, - }))); - core.execute_point_put( - &task, - PointPutExec { - tid: 1, - collection: "orders", - document_id: "o1", - surrogate: Surrogate::new(7), - value: &doc_value("a", "1"), - returning: None, - rls_filters: &[], - resolved_sum_targets: &[], - }, - ); - assert_eq!( - core.write_index.key_write_lsn(&surrogate_key("orders", 7)), - None - ); - assert_eq!(core.watermark, Lsn::ZERO); - } - - #[test] - fn horizon_gc_evicts_stale_keys_keeps_recent_and_collection() { - let (mut core, _, _, _dir) = make_core(); - let db = DatabaseId::DEFAULT; - let tenant = TenantId::new(1); - - // A stale per-key entry, then a recent write that drives the watermark far - // past the retain window. - core.note_write_lsn(db, tenant, "c", Some(KeyRepr::Surrogate(1)), Lsn::new(10)); - core.note_write_lsn( - db, - tenant, - "c", - Some(KeyRepr::Surrogate(2)), - Lsn::new(1_000_000), - ); - assert_eq!(core.watermark, Lsn::new(1_000_000)); - - core.gc_write_index(); - - // Stale key evicted; recent key retained; collection floor survives GC. - assert_eq!(core.write_index.key_write_lsn(&surrogate_key("c", 1)), None); - assert_eq!( - core.write_index.key_write_lsn(&surrogate_key("c", 2)), - Some(Lsn::new(1_000_000)) - ); - assert_eq!( - core.write_index.collection_write_lsn(&coll_key("c")), - Some(Lsn::new(1_000_000)) - ); - } - - // ── Read-set validation against the write-version index (integration, - // through `CoreLoop::read_set_still_current`) ────────────────────────── - - use nodedb_types::calvin::VersionedReadEntry; - - /// An `ExecutionTask` homing to `vshard_id`, carrying no WAL LSN. - fn task_with_vshard(vshard_id: VShardId) -> ExecutionTask { - ExecutionTask::new(Request { - vshard_id, - ..make_request(PhysicalPlan::Document(DocumentOp::PointGet { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "x"), - document_id: "y".into(), - surrogate: None, - pk_bytes: Vec::new(), - rls_filters: Vec::new(), - system_time: nodedb_types::SystemTimeScope::Current, - valid_at_ms: None, - })) - }) - } - - /// An `ExecutionTask` carrying WAL LSN `lsn` and homing to `vshard_id`. - fn wal_task_with_vshard(lsn: u64, vshard_id: VShardId) -> ExecutionTask { - ExecutionTask::with_wal_lsn( - Request { - vshard_id, - ..make_request(PhysicalPlan::Document(DocumentOp::PointGet { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "x"), - document_id: "y".into(), - surrogate: None, - pk_bytes: Vec::new(), - rls_filters: Vec::new(), - system_time: nodedb_types::SystemTimeScope::Current, - valid_at_ms: None, - })) - }, - Some(Lsn::new(lsn)), - ) - } - - /// The vShard `collection` homes to in the default database — mirrors the - /// homing `read_set_still_current` filters entries by. - fn local_vshard(collection: &str) -> VShardId { - nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, collection).vshard() - } - - /// Some vShard other than `than`, for exercising the cross-shard filter. - fn other_vshard(than: VShardId) -> VShardId { - VShardId::new((than.as_u32() + 1) % VShardId::COUNT) - } - - fn point_entry(collection: &str, surrogate: u32, read_lsn: u64) -> VersionedReadEntry { - VersionedReadEntry { - engine: EngineTag::Document, - collection: collection.to_string(), - key: ReadKeyIdent::Point(KeyRepr::Surrogate(surrogate)), - read_lsn: Lsn::new(read_lsn), - home_vshard: None, - served_by: 0, - } - } - - fn predicate_entry(collection: &str, read_lsn: u64) -> VersionedReadEntry { - VersionedReadEntry { - engine: EngineTag::Document, - collection: collection.to_string(), - key: ReadKeyIdent::Predicate, - read_lsn: Lsn::new(read_lsn), - home_vshard: None, - served_by: 0, - } - } - - #[test] - fn stale_point_read_is_detected_as_not_current() { - let (mut core, _, _, _dir) = make_core(); - core.note_write_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "orders", - Some(KeyRepr::Surrogate(7)), - Lsn::new(20), - ); - - let task = task_with_vshard(local_vshard("orders")); - let reads = vec![point_entry("orders", 7, 10)]; - assert!(!core.read_set_still_current(&task, 1, &reads)); - } - - #[test] - fn fresh_point_read_is_still_current() { - let (mut core, _, _, _dir) = make_core(); - core.note_write_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "orders", - Some(KeyRepr::Surrogate(7)), - Lsn::new(20), - ); - - let task = task_with_vshard(local_vshard("orders")); - assert!(core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 20)])); - assert!(core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 30)])); - } - - #[test] - fn read_entry_homing_to_a_different_vshard_is_filtered_out() { - let (mut core, _, _, _dir) = make_core(); - core.note_write_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "orders", - Some(KeyRepr::Surrogate(7)), - Lsn::new(20), - ); - - let local = local_vshard("orders"); - let remote_task = task_with_vshard(other_vshard(local)); - // Would conflict (read_lsn 10 < write_lsn 20) if this shard owned the - // entry's collection; it homes elsewhere, so it is filtered out of this - // shard's slice and the vacuous (empty-after-filter) result is `true`. - let reads = vec![point_entry("orders", 7, 10)]; - assert!(core.read_set_still_current(&remote_task, 1, &reads)); - } - - #[test] - fn stale_predicate_read_is_detected_as_not_current() { - let (mut core, _, _, _dir) = make_core(); - core.note_write_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "orders", - None, - Lsn::new(20), - ); - - let task = task_with_vshard(local_vshard("orders")); - let reads = vec![predicate_entry("orders", 10)]; - assert!(!core.read_set_still_current(&task, 1, &reads)); - } - - fn homed_entry(collection: &str, home: VShardId, read_lsn: u64) -> VersionedReadEntry { - VersionedReadEntry { - home_vshard: Some(home.as_u32()), - served_by: 0, - ..predicate_entry(collection, read_lsn) - } - } - - #[test] - fn a_homed_read_validates_on_its_home_not_its_collection_vshard() { - let (mut core, _, _, _dir) = make_core(); - core.note_write_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "edges", - None, - Lsn::new(20), - ); - let home = other_vshard(local_vshard("edges")); - let stale = vec![homed_entry("edges", home, 10)]; - assert!(!core.read_set_still_current(&task_with_vshard(home), 1, &stale)); - assert!( - core.read_set_still_current(&task_with_vshard(local_vshard("edges")), 1, &stale), - "the collection's own vShard does not hold a homed read" - ); - let fresh = vec![homed_entry("edges", home, 20)]; - assert!(core.read_set_still_current(&task_with_vshard(home), 1, &fresh)); - } - - #[test] - fn a_homed_read_of_every_collection_validates_against_the_core_watermark() { - let (mut core, _, _, _dir) = make_core(); - let home = local_vshard("edges"); - core.note_write_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "unrelated", - None, - Lsn::new(20), - ); - let task = task_with_vshard(home); - assert!(!core.read_set_still_current(&task, 1, &[homed_entry("", home, 10)])); - assert!(core.read_set_still_current(&task, 1, &[homed_entry("", home, 20)])); - } - - #[test] - fn fresh_predicate_read_is_still_current() { - let (mut core, _, _, _dir) = make_core(); - core.note_write_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "orders", - None, - Lsn::new(20), - ); - - let task = task_with_vshard(local_vshard("orders")); - let reads = vec![predicate_entry("orders", 20)]; - assert!(core.read_set_still_current(&task, 1, &reads)); - } - - #[test] - fn empty_read_set_is_vacuously_current() { - let (core, _, _, _dir) = make_core(); - let task = task_with_vshard(VShardId::new(0)); - assert!(core.read_set_still_current(&task, 1, &[])); - } - - #[test] - fn a_read_that_predates_a_committed_write_is_no_longer_current() { - let (mut core, _, _, _dir) = make_core(); - let vshard = local_vshard("orders"); - - // A committed write to key 7 in "orders" records its version at LSN 10. - let write_task = wal_task_with_vshard(10, vshard); - let write_plans = vec![PhysicalPlan::Document(DocumentOp::PointPut { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "orders"), - document_id: "o7".into(), - value: doc_value("a", "1"), - surrogate: Surrogate::new(7), - pk_bytes: Vec::new(), - returning: None, - rls_filters: Vec::new(), - resolved_sum_targets: Vec::new(), - })]; - let write_resp = core.commit_plans_for_test(&write_task, 1, &write_plans, 10); - assert_eq!(write_resp.status, Status::Ok, "{:?}", write_resp.error_code); - - // A read of key 7 observed at LSN 5 predates the write; one observed - // at LSN 10 does not. - let task = task_with_vshard(vshard); - assert!(!core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 5)])); - assert!(core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 10)])); - } -} diff --git a/nodedb/src/data/executor/core_loop/write_index/index.rs b/nodedb/src/data/executor/core_loop/write_index/index.rs new file mode 100644 index 000000000..7fd5d7136 --- /dev/null +++ b/nodedb/src/data/executor/core_loop/write_index/index.rs @@ -0,0 +1,544 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The per-core write-version index and its read validation. + +use std::collections::HashMap; + +use nodedb_types::calvin::ReadKeyIdent; +use nodedb_types::{DatabaseId, TenantId, WriteVersion}; + +use crate::types::VShardId; + +use super::super::index_value_versions::{IndexDimRef, IndexValueVersionIndex, IndexVersion}; +use super::keys::{CollKey, KeyRepr, WriteKey}; +use super::record_stamps::RecordStamps; + +/// Horizon retain window for per-key entries. Horizon GC evicts a +/// `last_write` entry more than this many log entries (or WAL LSNs, for +/// writes that apply no entry) below its vShard's latest version. Sized in +/// the same order of magnitude as the idempotency-cache cap (16,384 +/// entries): a bounded recent-write history, enough to validate in-flight +/// transactions, not an unbounded write log. +pub const RETAIN_WINDOW: u64 = 16_384; + +/// Hard upper bound on the `last_write` entry count. When horizon GC leaves +/// more entries than this, the lowest versions are dropped until the map is +/// back within bound. +const MAX_KEY_ENTRIES: usize = 65_536; + +/// One vShard's version bounds on this core. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct VShardBounds { + /// The highest version recorded for the vShard. + latest: WriteVersion, + /// The version the vShard's state holds without per-row versions: a + /// snapshot install's cut, or the newest WAL record restart replay read. + installed: WriteVersion, + /// The highest per-key or per-value version horizon GC evicted. + evicted: WriteVersion, +} + +/// Per-core write-version index. +/// +/// Every version is a position in one vShard's history, so every map is +/// keyed by vShard and versions of two vShards are never compared. +#[derive(Default)] +pub struct WriteVersionIndex { + /// Last committed-write version per written key. + last_write: HashMap, + /// Last committed-write version per written collection (the phantom-safe + /// floor: a predicate reader validates against it). Never GC'd: bounded + /// by the live collections of the core's vShards. + coll_write: HashMap, + /// Per-secondary-index-dimension write-VALUE versions, the finer-grained + /// sibling of `coll_write` an index-range read validates against. + pub(in crate::data::executor) index_values: IndexValueVersionIndex, + /// Per-vShard bounds. + vshards: HashMap, + /// The stamps of the WAL records the replay arms apply: every record + /// during restart replay, the installed record during a committed redo + /// install. + pub(in crate::data::executor) record_stamps: RecordStamps, +} + +impl WriteVersionIndex { + pub fn new() -> Self { + Self::default() + } + + /// Record a committed write at `version` on `vshard`. + /// + /// Advances the collection's version and, when `key` is `Some`, the + /// key's version, each to the max of its current value and `version`. + /// Raises the vShard's latest version the same way. + pub fn note_write( + &mut self, + db: DatabaseId, + tenant: TenantId, + vshard: VShardId, + collection: &str, + key: Option, + version: WriteVersion, + ) { + let coll_key = CollKey { + vshard, + db, + tenant, + collection: Box::from(collection), + }; + let slot = self.coll_write.entry(coll_key).or_default(); + *slot = (*slot).max(version); + + if let Some(key) = key { + let write_key = WriteKey { + vshard, + db, + tenant, + collection: Box::from(collection), + key, + }; + let slot = self.last_write.entry(write_key).or_default(); + *slot = (*slot).max(version); + } + self.raise_latest(vshard, version); + } + + /// Raise `vshard`'s latest version to `version` when it is higher. + pub(in crate::data::executor) fn raise_latest( + &mut self, + vshard: VShardId, + version: WriteVersion, + ) { + let bounds = self.vshards.entry(vshard).or_default(); + bounds.latest = bounds.latest.max(version); + } + + /// The vShard's state now holds every write through `version`, with no + /// per-row versions: a snapshot install landed it. Every recorded version + /// of the vShard is dropped, because the install replaced the rows they + /// describe. A read older than `version` no longer validates. + pub fn install(&mut self, vshard: VShardId, version: WriteVersion) { + self.last_write.retain(|key, _| key.vshard != vshard); + self.coll_write.retain(|key, _| key.vshard != vshard); + self.index_values.drop_vshard(vshard); + let bounds = self.vshards.entry(vshard).or_default(); + bounds.installed = bounds.installed.max(version); + bounds.latest = bounds.latest.max(version); + } + + /// The vShard's state holds writes through `version` that no per-row + /// version describes: restart replay read records it did not apply. A + /// read older than `version` no longer validates against a row with no + /// recorded version. + pub fn raise_installed(&mut self, vshard: VShardId, version: WriteVersion) { + let bounds = self.vshards.entry(vshard).or_default(); + bounds.installed = bounds.installed.max(version); + bounds.latest = bounds.latest.max(version); + } + + /// Whether this core recorded any version of `vshard`. + pub(in crate::data::executor) fn holds(&self, vshard: VShardId) -> bool { + self.vshards.contains_key(&vshard) + } + + /// The highest version recorded for `vshard`, `ZERO` when none is. + pub(in crate::data::executor) fn latest(&self, vshard: VShardId) -> WriteVersion { + self.vshards + .get(&vshard) + .map(|bounds| bounds.latest) + .unwrap_or_default() + } + + /// Every vShard this core recorded a version of, with its latest one. + pub(in crate::data::executor) fn latest_by_vshard( + &self, + ) -> impl Iterator + '_ { + self.vshards + .iter() + .map(|(vshard, bounds)| (*vshard, bounds.latest)) + } + + /// Recorded per-key version, if any. + #[cfg(test)] + pub(crate) fn key_version(&self, key: &WriteKey) -> Option { + self.last_write.get(key).copied() + } + + /// Recorded per-collection version, if any. + #[cfg(test)] + pub(crate) fn collection_version(&self, key: &CollKey) -> Option { + self.coll_write.get(key).copied() + } + + /// Every recorded per-key version. + #[cfg(test)] + pub(crate) fn recorded_keys(&self) -> impl Iterator { + self.last_write.iter().map(|(key, version)| (key, *version)) + } + + /// Every recorded per-collection version. + #[cfg(test)] + pub(crate) fn recorded_collections(&self) -> impl Iterator { + self.coll_write.iter().map(|(key, version)| (key, *version)) + } + + /// The version of `collection` on `vshard` a read compares against: the + /// recorded one, else the version the vShard's state was installed at. + pub(in crate::data::executor) fn collection_current( + &self, + db: DatabaseId, + tenant: TenantId, + vshard: VShardId, + collection: &str, + ) -> WriteVersion { + let key = CollKey { + vshard, + db, + tenant, + collection: Box::from(collection), + }; + self.coll_write + .get(&key) + .copied() + .unwrap_or_else(|| self.bounds(vshard).installed) + } + + fn bounds(&self, vshard: VShardId) -> VShardBounds { + self.vshards.get(&vshard).copied().unwrap_or_default() + } + + /// Whether a read of `collection` on `vshard`, observed at `read`, is + /// still current. + /// + /// A `Point` read is current iff the key's version is at or below the + /// read's. A `Predicate` read compares the collection's version. An + /// index read compares the versions of the values it observed, and falls + /// back to the collection's version for a dimension no write recorded. + /// + /// A key or value with no recorded version was not written since the + /// vShard was installed, or GC evicted it. Its version is then at most + /// the installed or evicted bound, and never above the collection's. + pub(crate) fn read_is_valid( + &self, + db: DatabaseId, + tenant: TenantId, + vshard: VShardId, + collection: &str, + key: &ReadKeyIdent, + read: WriteVersion, + ) -> bool { + let collection_current = self.collection_current(db, tenant, vshard, collection); + let bounds = self.bounds(vshard); + let unrecorded = collection_current.min(bounds.installed.max(bounds.evicted)); + match key { + ReadKeyIdent::Point(repr) => { + let write_key = WriteKey { + vshard, + db, + tenant, + collection: Box::from(collection), + key: repr.clone(), + }; + self.last_write + .get(&write_key) + .copied() + .unwrap_or(unrecorded) + <= read + } + ReadKeyIdent::Predicate => collection_current <= read, + ReadKeyIdent::IndexEq { field, value } => { + let dim = IndexDimRef { + vshard, + db, + tenant, + collection, + field, + }; + match self.index_values.eq_version(dim, value) { + IndexVersion::Untracked => collection_current <= read, + IndexVersion::Tracked(recorded) => recorded.unwrap_or(unrecorded) <= read, + } + } + ReadKeyIdent::IndexRange { field, lo, hi } => { + let dim = IndexDimRef { + vshard, + db, + tenant, + collection, + field, + }; + match self + .index_values + .range_version(dim, lo.as_deref(), hi.as_deref()) + { + IndexVersion::Untracked => collection_current <= read, + // A value GC evicted from the range is unknown, so the + // range takes the unrecorded bound as well. + IndexVersion::Tracked(recorded) => { + recorded.unwrap_or_default().max(unrecorded) <= read + } + } + } + } + } + + /// Horizon garbage-collect the per-key and per-value maps. + /// + /// Evicts every entry more than [`RETAIN_WINDOW`] below its vShard's + /// latest version in the same epoch, then, if more than + /// [`MAX_KEY_ENTRIES`] per-key entries remain, drops the lowest versions + /// until back within bound. Each vShard's evicted bound rises to the + /// highest version it lost. The per-collection map is left untouched. + pub fn gc(&mut self) { + let latest: HashMap = self.latest_by_vshard().collect(); + let mut evicted: HashMap = HashMap::new(); + self.last_write.retain(|key, version| { + let keep = latest + .get(&key.vshard) + .and_then(|latest| latest.distance_above(*version)) + .is_none_or(|distance| distance <= RETAIN_WINDOW); + if !keep { + let slot = evicted.entry(key.vshard).or_default(); + *slot = (*slot).max(*version); + } + keep + }); + + if self.last_write.len() > MAX_KEY_ENTRIES { + let overflow = self.last_write.len() - MAX_KEY_ENTRIES; + let mut by_version: Vec<(WriteVersion, WriteKey)> = self + .last_write + .iter() + .map(|(key, version)| (*version, key.clone())) + .collect(); + // TOTAL order so tied-version eviction is replica-identical: a + // plain sort by version over a `HashMap`-collected Vec would let + // the dropped set depend on hash-iteration layout. `DatabaseId` / + // `TenantId` lack `Ord` (compare via `as_u64()`); `KeyRepr` is + // `Ord`. + by_version.sort_by(|a, b| { + a.0.cmp(&b.0) + .then_with(|| a.1.vshard.cmp(&b.1.vshard)) + .then_with(|| a.1.db.as_u64().cmp(&b.1.db.as_u64())) + .then_with(|| a.1.tenant.as_u64().cmp(&b.1.tenant.as_u64())) + .then_with(|| a.1.collection.cmp(&b.1.collection)) + .then_with(|| a.1.key.cmp(&b.1.key)) + }); + for (version, key) in by_version.into_iter().take(overflow) { + let slot = evicted.entry(key.vshard).or_default(); + *slot = (*slot).max(version); + self.last_write.remove(&key); + } + } + + for (vshard, version) in self.index_values.gc(&latest) { + let slot = evicted.entry(vshard).or_default(); + *slot = (*slot).max(version); + } + for (vshard, version) in evicted { + let bounds = self.vshards.entry(vshard).or_default(); + bounds.evicted = bounds.evicted.max(version); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn db() -> DatabaseId { + DatabaseId::DEFAULT + } + + fn tenant() -> TenantId { + TenantId::new(1) + } + + fn vshard() -> VShardId { + VShardId::new(5) + } + + fn at(index: u64) -> WriteVersion { + WriteVersion::logged(0, index) + } + + fn point(surrogate: u32) -> ReadKeyIdent { + ReadKeyIdent::Point(KeyRepr::Surrogate(surrogate)) + } + + fn valid(index: &WriteVersionIndex, key: &ReadKeyIdent, read: u64) -> bool { + index.read_is_valid(db(), tenant(), vshard(), "orders", key, at(read)) + } + + fn write(index: &mut WriteVersionIndex, key: Option, version: u64) { + index.note_write(db(), tenant(), vshard(), "orders", key, at(version)); + } + + fn record_value(index: &mut WriteVersionIndex, field: &str, value: &str, version: u64) { + let dim = IndexDimRef { + vshard: vshard(), + db: db(), + tenant: tenant(), + collection: "orders", + field, + }; + index.index_values.record(dim, value, at(version)); + } + + #[test] + fn point_read_is_valid_when_key_never_written() { + let index = WriteVersionIndex::new(); + assert!(valid(&index, &point(7), 10)); + } + + #[test] + fn point_read_is_valid_when_write_at_or_before_read() { + let mut index = WriteVersionIndex::new(); + write(&mut index, Some(KeyRepr::Surrogate(7)), 10); + assert!(valid(&index, &point(7), 10)); + assert!(valid(&index, &point(7), 20)); + } + + #[test] + fn point_read_is_invalid_when_write_after_read() { + let mut index = WriteVersionIndex::new(); + write(&mut index, Some(KeyRepr::Surrogate(7)), 20); + assert!(!valid(&index, &point(7), 10)); + } + + #[test] + fn predicate_read_follows_the_collection_version() { + let mut index = WriteVersionIndex::new(); + assert!(valid(&index, &ReadKeyIdent::Predicate, 10)); + write(&mut index, None, 10); + assert!(valid(&index, &ReadKeyIdent::Predicate, 10)); + assert!(valid(&index, &ReadKeyIdent::Predicate, 20)); + write(&mut index, None, 20); + assert!(!valid(&index, &ReadKeyIdent::Predicate, 10)); + } + + #[test] + fn a_write_on_another_vshard_never_moves_a_read() { + let mut index = WriteVersionIndex::new(); + index.note_write( + db(), + tenant(), + VShardId::new(6), + "orders", + Some(KeyRepr::Surrogate(7)), + at(900), + ); + assert!(valid(&index, &point(7), 10)); + assert!(valid(&index, &ReadKeyIdent::Predicate, 10)); + } + + #[test] + fn untracked_index_dimension_falls_back_to_collection_version() { + let index_eq = ReadKeyIdent::IndexEq { + field: "email".to_string(), + value: "a@b.c".to_string(), + }; + let index_range = ReadKeyIdent::IndexRange { + field: "age".to_string(), + lo: Some("18".to_string()), + hi: None, + }; + for (floor, read) in [(5u64, 10u64), (10, 10), (20, 10)] { + let mut index = WriteVersionIndex::new(); + write(&mut index, None, floor); + let want = valid(&index, &ReadKeyIdent::Predicate, read); + assert_eq!(valid(&index, &index_eq, read), want, "eq {floor} {read}"); + assert_eq!( + valid(&index, &index_range, read), + want, + "range {floor} {read}" + ); + } + } + + #[test] + fn a_disjoint_index_value_write_does_not_abort() { + let mut index = WriteVersionIndex::new(); + write(&mut index, None, 20); + record_value(&mut index, "email", "z@z.z", 20); + record_value(&mut index, "age", "50", 20); + let eq = ReadKeyIdent::IndexEq { + field: "email".to_string(), + value: "a@b.c".to_string(), + }; + let range = ReadKeyIdent::IndexRange { + field: "age".to_string(), + lo: Some("10".to_string()), + hi: Some("20".to_string()), + }; + assert!(valid(&index, &eq, 10)); + assert!(valid(&index, &range, 10)); + } + + #[test] + fn an_index_value_written_after_the_read_aborts() { + let mut index = WriteVersionIndex::new(); + record_value(&mut index, "email", "a@b.c", 20); + record_value(&mut index, "age", "15", 20); + let eq = ReadKeyIdent::IndexEq { + field: "email".to_string(), + value: "a@b.c".to_string(), + }; + let range = ReadKeyIdent::IndexRange { + field: "age".to_string(), + lo: Some("10".to_string()), + hi: Some("20".to_string()), + }; + assert!(!valid(&index, &eq, 10)); + assert!(!valid(&index, &range, 10)); + } + + #[test] + fn a_phantom_insert_into_a_read_range_aborts() { + let mut index = WriteVersionIndex::new(); + record_value(&mut index, "age", "99", 5); + let range = ReadKeyIdent::IndexRange { + field: "age".to_string(), + lo: Some("10".to_string()), + hi: Some("20".to_string()), + }; + assert!(valid(&index, &range, 10), "no in-range write"); + record_value(&mut index, "age", "15", 20); + assert!(!valid(&index, &range, 10), "phantom in-range insert"); + } + + #[test] + fn horizon_gc_evicts_stale_keys_and_an_evicted_key_bounds_old_reads() { + let mut index = WriteVersionIndex::new(); + write(&mut index, Some(KeyRepr::Surrogate(1)), 10); + write(&mut index, Some(KeyRepr::Surrogate(2)), 1_000_000); + index.gc(); + + let key = |surrogate| WriteKey { + vshard: vshard(), + db: db(), + tenant: tenant(), + collection: Box::from("orders"), + key: KeyRepr::Surrogate(surrogate), + }; + assert_eq!(index.key_version(&key(1)), None); + assert_eq!(index.key_version(&key(2)), Some(at(1_000_000))); + // The evicted key's version is unknown, so a read older than the + // eviction bound no longer validates; a newer one does. + assert!(!valid(&index, &point(1), 5)); + assert!(valid(&index, &point(1), 10)); + } + + #[test] + fn an_install_drops_the_vshards_versions_and_bounds_every_read() { + let mut index = WriteVersionIndex::new(); + write(&mut index, Some(KeyRepr::Surrogate(7)), 10); + index.install(vshard(), at(50)); + assert_eq!(index.latest(vshard()), at(50)); + // The install replaced key 7: its version is unknown up to the cut. + assert!(!valid(&index, &point(7), 20)); + assert!(!valid(&index, &ReadKeyIdent::Predicate, 20)); + assert!(valid(&index, &point(7), 50)); + assert!(valid(&index, &ReadKeyIdent::Predicate, 50)); + } +} diff --git a/nodedb/src/data/executor/core_loop/write_index/keys.rs b/nodedb/src/data/executor/core_loop/write_index/keys.rs new file mode 100644 index 000000000..c0bfcbbfc --- /dev/null +++ b/nodedb/src/data/executor/core_loop/write_index/keys.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The keys the write-version index records versions under, and the stamp a +//! committed write records them with. + +use nodedb_types::{DatabaseId, TenantId, WriteVersion}; + +use crate::types::{Lsn, VShardId}; + +/// Row identity type, re-exported from its plane-neutral home +/// ([`crate::types::KeyRepr`]) so Data-Plane call sites can keep referring to +/// it through this module. Read keys and write keys share this one namespace. +pub use crate::types::KeyRepr; + +/// Fully-qualified per-key version-index key. Scoped by `(database, tenant)` +/// exactly like the write path, so two tenants (or databases) never alias. +/// A version positions a write within one vShard, so the vShard is part of +/// the key: an edge written on two endpoint vShards holds one version on +/// each. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct WriteKey { + pub vshard: VShardId, + pub db: DatabaseId, + pub tenant: TenantId, + pub collection: Box, + pub key: KeyRepr, +} + +/// Fully-qualified per-collection version-index key, on one vShard. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CollKey { + pub vshard: VShardId, + pub db: DatabaseId, + pub tenant: TenantId, + pub collection: Box, +} + +/// What the version index records one committed write under. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WriteStamp { + /// The vShard whose history the write extends. + pub vshard: VShardId, + /// The LSN of the write's WAL record on this node. + pub lsn: Lsn, + /// The log position of the data-group entry the write applies. `None` + /// for a write that applies no entry. + pub entry: Option, +} diff --git a/nodedb/src/data/executor/core_loop/write_index/mod.rs b/nodedb/src/data/executor/core_loop/write_index/mod.rs new file mode 100644 index 000000000..e6756e9c0 --- /dev/null +++ b/nodedb/src/data/executor/core_loop/write_index/mod.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Per-core write-version index. +//! +//! Records, for every committed write applied on this Data-Plane core, the +//! version of the write against the written key and against the written +//! collection, on the write's vShard. A write that applies a data-group Raft +//! entry takes the entry's log position as its version, so every replica of +//! the vShard records the same version for it. A write that applies no entry +//! takes its WAL LSN on top of the vShard's latest version. +//! +//! This is the shard-local write-version substrate the optimistic-concurrency +//! commit path validates a transaction's read-set against (see +//! `CoreLoop::read_set_still_current`). Because the index lives on the +//! `!Send` core it is a plain `HashMap`: no atomics, no locks, no cross-core +//! sharing. +//! +//! The per-key map is bounded: horizon GC (run from the periodic maintenance +//! hook) evicts entries far below their vShard's latest version and enforces +//! a hard entry-count backstop. The per-collection map is bounded by the +//! number of live collections and is never GC'd. + +pub mod index; +pub mod keys; +mod record; +pub mod record_stamps; +#[cfg(test)] +pub mod tests; +mod validate; + +pub use index::{RETAIN_WINDOW, WriteVersionIndex}; +#[cfg(test)] +pub use keys::{CollKey, WriteKey}; +pub use keys::{KeyRepr, WriteStamp}; +pub use record_stamps::RecordStamps; diff --git a/nodedb/src/data/executor/core_loop/write_index/record.rs b/nodedb/src/data/executor/core_loop/write_index/record.rs new file mode 100644 index 000000000..6b54575a7 --- /dev/null +++ b/nodedb/src/data/executor/core_loop/write_index/record.rs @@ -0,0 +1,548 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The apply chokepoints that record committed writes into the per-core +//! write-version index. + +use nodedb_types::{DatabaseId, ShardVersion, TenantId, WriteVersion}; + +use crate::data::executor::task::ExecutionTask; +use crate::types::{Lsn, VShardId}; + +use super::super::CoreLoop; +use super::keys::{KeyRepr, WriteStamp}; + +impl CoreLoop { + /// The stamp of the write `task` applies: its vShard, its WAL record and + /// the log position of the data-group entry it applies. `None` for a task + /// that carries no WAL record. A record replay or a committed redo install + /// applies takes the stamp that record names. + pub(in crate::data::executor) fn task_write_stamp( + &self, + task: &ExecutionTask, + ) -> Option { + task.wal_lsn() + .map(|lsn| self.task_write_stamp_at(task, lsn)) + } + + /// The stamp of the write `task` applies from the WAL record at `lsn`, + /// for a handler that reads the record's LSN off its plan. + pub(in crate::data::executor) fn task_write_stamp_at( + &self, + task: &ExecutionTask, + lsn: Lsn, + ) -> WriteStamp { + self.write_index.record_stamps.resolve(WriteStamp { + vshard: task.request.vshard_id, + lsn, + entry: task.request.entry_version, + }) + } + + /// The version a write with `stamp` records: the log position of the + /// entry it applies, or, for a write that applies no entry, its WAL LSN on + /// top of the latest version its vShard holds here. + pub(in crate::data::executor) fn write_version_of(&self, stamp: WriteStamp) -> WriteVersion { + stamp.entry.unwrap_or_else(|| { + WriteVersion::local_after(self.write_index.latest(stamp.vshard), stamp.lsn.as_u64()) + }) + } + + /// Record a committed write into the per-core version index and advance + /// the core watermark monotonically. + /// + /// Called once per written key at every Data-Plane apply chokepoint. `key` + /// is `None` for engines whose per-key identity is internal (columnar / + /// timeseries / array / spatial / FTS): those record only the collection + /// version. + /// + /// In the install pass of a committed-redo apply the version waits in the + /// apply scope. The apply publishes it once the record settled, so a + /// rolled-back install leaves no version and no watermark behind. + pub(in crate::data::executor) fn note_write( + &mut self, + db: DatabaseId, + tenant: TenantId, + collection: &str, + key: Option, + stamp: WriteStamp, + ) { + if let Some(scope) = self.redo_apply.scope.as_mut() { + scope.defer_write_version(db, tenant, collection, key, stamp); + return; + } + self.publish_write_version(db, tenant, collection, key, stamp); + } + + /// Record a write version and advance the core watermark monotonically. + /// + /// Every applied write reaches here, so this is also where the core notes + /// the write's LSN as applied: a checkpoint's replay stamp names it. + pub(in crate::data::executor) fn publish_write_version( + &mut self, + db: DatabaseId, + tenant: TenantId, + collection: &str, + key: Option, + stamp: WriteStamp, + ) { + self.floors.applied_prefix.note_applied(stamp.lsn); + let version = self.write_version_of(stamp); + self.write_index + .note_write(db, tenant, stamp.vshard, collection, key, version); + if stamp.lsn > self.watermark { + self.watermark = stamp.lsn; + } + } + + /// Record a committed write's collection version only (no per-key + /// entry), if `task` carries a WAL record. Shared by the columnar-family + /// write handlers (columnar / timeseries / array / spatial / FTS) whose + /// per-key identity is internal: a predicate reader validates against + /// the collection version when it owns no per-key version. + pub(in crate::data::executor) fn note_collection_write( + &mut self, + task: &ExecutionTask, + collection: &str, + ) { + if let Some(stamp) = self.task_write_stamp(task) { + self.note_write( + task.request.database_id, + task.request.tenant_id, + collection, + None, + stamp, + ); + } + } + + /// Record a committed document/vector write's version, keyed by the + /// written row's cross-engine surrogate, if `task` carries a WAL record. + /// Shared by every per-surrogate write chokepoint (point put, point + /// insert, point delete, bulk update, bulk delete). + pub(in crate::data::executor) fn note_surrogate_write( + &mut self, + task: &ExecutionTask, + tid: u64, + collection: &str, + surrogate: u32, + ) { + if let Some(stamp) = self.task_write_stamp(task) { + self.note_write( + task.request.database_id, + TenantId::new(tid), + collection, + Some(KeyRepr::Surrogate(surrogate)), + stamp, + ); + } + } + + /// Record the version of a write a replay arm applied from the WAL record + /// at `record_lsn`. A no-op when `record_lsn == 0` (no durable record). + /// `key` is `None` for collection-only entries (e.g. truncate) and `Some` + /// for per-key/per-surrogate entries, exactly like [`Self::note_write`]. + /// + /// The replay arms run for restart replay and for a committed redo + /// install. Both register the stamp of the record they apply, so the + /// write takes that record's vShard and entry position. A record nothing + /// registered applies on its collection's home vShard, the vShard a live + /// write of the collection routes to, and names no entry. + pub(in crate::data::executor) fn note_replay_write( + &mut self, + database_id: u64, + tenant_id: u64, + collection: &str, + key: Option, + record_lsn: u64, + ) { + if record_lsn == 0 { + return; + } + let db = DatabaseId::new(database_id); + let stamp = match self.write_index.record_stamps.get(record_lsn) { + Some(stamp) => stamp, + None => WriteStamp { + // A name without its database's qualifier is a bare name. + vshard: nodedb_types::CollectionKey::from_qualified_str(db, collection) + .unwrap_or_else(|_| nodedb_types::CollectionKey::from_bare(db, collection)) + .vshard(), + lsn: Lsn::new(record_lsn), + entry: None, + }, + }; + self.note_write(db, TenantId::new(tenant_id), collection, key, stamp); + } + + /// Run horizon GC on the per-core version index. Invoked from the periodic + /// maintenance hook: no dedicated timer. + pub(in crate::data::executor) fn gc_write_index(&mut self) { + self.write_index.gc(); + } + + /// Land a snapshot install's cut on each of `floor`'s vShards: the + /// installed rows hold every write through the cut and carry no per-row + /// versions. + pub(in crate::data::executor) fn install_version_floor(&mut self, floor: &[ShardVersion]) { + for shard in floor { + if shard.vshard < VShardId::COUNT { + self.write_index + .install(VShardId::new(shard.vshard), shard.version); + } + } + } +} + +#[cfg(test)] +mod tests { + use nodedb_physical::physical_plan::DocumentOp; + use nodedb_types::{QualifiedCollection, Surrogate}; + + use super::super::keys::{CollKey, WriteKey}; + use super::super::tests::{ + doc_value, entry_task, local, make_core, make_request, point_put, wal_task, + }; + use super::*; + use crate::bridge::envelope::{PhysicalPlan, Status}; + use crate::data::executor::handlers::graph::EdgePutParams; + use crate::data::executor::handlers::kv::crud::KvWriteParams; + + fn vshard0() -> VShardId { + VShardId::new(0) + } + + fn surrogate_key(collection: &str, surrogate: u32) -> WriteKey { + WriteKey { + vshard: vshard0(), + db: DatabaseId::DEFAULT, + tenant: TenantId::new(1), + collection: Box::from(collection), + key: KeyRepr::Surrogate(surrogate), + } + } + + fn coll_key(collection: &str) -> CollKey { + CollKey { + vshard: vshard0(), + db: DatabaseId::DEFAULT, + tenant: TenantId::new(1), + collection: Box::from(collection), + } + } + + #[test] + fn a_write_with_no_entry_records_a_local_version_and_advances_the_watermark() { + let (mut core, _, _, _dir) = make_core(); + + let resp = point_put(&mut core, &wal_task(10), "orders", "o1", 7, "1"); + assert_eq!(resp.status, Status::Ok); + assert_eq!( + core.write_index.key_version(&surrogate_key("orders", 7)), + Some(local(10)) + ); + assert_eq!( + core.write_index.collection_version(&coll_key("orders")), + Some(local(10)) + ); + assert_eq!(core.watermark, Lsn::new(10)); + + point_put(&mut core, &wal_task(20), "orders", "o1", 7, "2"); + assert_eq!( + core.write_index.key_version(&surrogate_key("orders", 7)), + Some(local(20)) + ); + assert_eq!(core.watermark, Lsn::new(20)); + + // A lower LSN never regresses an existing entry or the watermark. + point_put(&mut core, &wal_task(15), "orders", "o1", 7, "3"); + assert_eq!( + core.write_index.key_version(&surrogate_key("orders", 7)), + Some(local(20)) + ); + assert_eq!(core.watermark, Lsn::new(20)); + + // A second collection tracks its own version independently. + point_put(&mut core, &wal_task(30), "items", "i1", 9, "4"); + assert_eq!( + core.write_index.collection_version(&coll_key("items")), + Some(local(30)) + ); + assert_eq!( + core.write_index.collection_version(&coll_key("orders")), + Some(local(20)) + ); + assert_eq!(core.watermark, Lsn::new(30)); + } + + #[test] + fn the_version_recorded_is_the_applying_entrys_log_position() { + let (mut core, _, _, _dir) = make_core(); + let entry = WriteVersion::logged(4, 812); + + // The WAL LSN is this node's own. The version is the entry's position. + let resp = point_put(&mut core, &entry_task(57, entry), "orders", "o1", 7, "1"); + assert_eq!(resp.status, Status::Ok); + assert_eq!( + core.write_index.key_version(&surrogate_key("orders", 7)), + Some(entry) + ); + assert_eq!( + core.write_index.collection_version(&coll_key("orders")), + Some(entry) + ); + assert_eq!(core.write_index.latest(vshard0()), entry); + assert_eq!(core.watermark, Lsn::new(57)); + + // A replica that applies the same entry at another WAL LSN records + // the same version. + let (mut replica, _, _, _replica_dir) = make_core(); + point_put( + &mut replica, + &entry_task(9_001, entry), + "orders", + "o1", + 7, + "1", + ); + assert_eq!( + replica.write_index.key_version(&surrogate_key("orders", 7)), + Some(entry) + ); + } + + #[test] + fn a_point_update_records_the_rows_version() { + let (mut core, _, _, _dir) = make_core(); + point_put( + &mut core, + &entry_task(40, WriteVersion::logged(1, 5)), + "orders", + "o1", + 7, + "1", + ); + let raised = nodedb_types::json_to_msgpack(&serde_json::json!("2")).expect("encode"); + let updates = vec![( + "a".to_string(), + nodedb_physical::physical_plan::UpdateValue::Literal(raised), + )]; + let updated = WriteVersion::logged(1, 9); + let resp = core.execute_point_update( + &entry_task(41, updated), + crate::data::executor::handlers::point::update::PointUpdateParams { + tid: 1, + collection: "orders", + document_id: "o1", + surrogate: Some(Surrogate::new(7)), + updates: &updates, + returning: None, + rls_filters: &[], + rls_write_check: &nodedb_types::RlsWriteCheck::NoPolicyApplies, + resolved_sum_targets: &[], + declared_primary_key: None, + }, + ); + assert_eq!(resp.status, Status::Ok, "{:?}", resp.error_code); + assert_eq!( + core.write_index.key_version(&surrogate_key("orders", 7)), + Some(updated), + "a read of the row from before the update must fail validation" + ); + assert_eq!( + core.write_index.collection_version(&coll_key("orders")), + Some(updated) + ); + } + + #[test] + fn a_local_write_sorts_between_the_entries_around_it() { + let (mut core, _, _, _dir) = make_core(); + point_put( + &mut core, + &entry_task(40, WriteVersion::logged(0, 5)), + "orders", + "o1", + 7, + "1", + ); + point_put(&mut core, &wal_task(41), "orders", "o2", 8, "1"); + let local_write = core + .write_index + .key_version(&surrogate_key("orders", 8)) + .expect("local write recorded"); + assert_eq!( + local_write, + WriteVersion::local_after(WriteVersion::logged(0, 5), 41) + ); + assert!(local_write > WriteVersion::logged(0, 5)); + assert!(local_write < WriteVersion::logged(0, 6)); + } + + #[test] + fn kv_put_records_kvkey_version() { + let (mut core, _, _, _dir) = make_core(); + let resp = core.execute_kv_put( + &wal_task(42), + KvWriteParams { + did: DatabaseId::DEFAULT.as_u64(), + tid: 1, + collection: "kv", + key: b"k1".as_slice(), + value: b"v1".as_slice(), + ttl_ms: 0, + surrogate: Surrogate::new(3), + returning: None, + rls_filters: &[], + }, + ); + assert_eq!(resp.status, Status::Ok); + + let wk = WriteKey { + vshard: vshard0(), + db: DatabaseId::DEFAULT, + tenant: TenantId::new(1), + collection: Box::from("kv"), + key: KeyRepr::KvKey(Box::from(b"k1".as_slice())), + }; + assert_eq!(core.write_index.key_version(&wk), Some(local(42))); + assert_eq!( + core.write_index.collection_version(&coll_key("kv")), + Some(local(42)) + ); + assert_eq!(core.watermark, Lsn::new(42)); + } + + #[test] + fn an_autocommit_apply_is_named_by_the_next_replay_stamp() { + let (mut core, _, _, _dir) = make_core(); + core.floors + .applied_prefix + .observe_outcome_floor(Lsn::new(40)); + let resp = core.execute_kv_put( + &wal_task(42), + KvWriteParams { + did: DatabaseId::DEFAULT.as_u64(), + tid: 1, + collection: "kv", + key: b"k1".as_slice(), + value: b"v1".as_slice(), + ttl_ms: 0, + surrogate: Surrogate::new(3), + returning: None, + rls_filters: &[], + }, + ); + assert_eq!(resp.status, Status::Ok); + + let stamp = core.floors.applied_prefix.stamp().expect("exact stamp"); + assert!( + stamp.skips(42), + "the applied write is in every later artifact" + ); + assert!( + !stamp.skips(41), + "a record between the floor and the applied write still replays" + ); + } + + #[test] + fn edge_put_records_edge_version() { + let (mut core, _, _, _dir) = make_core(); + let resp = core.execute_edge_put( + &wal_task(50), + EdgePutParams { + tid: 1, + collection: "graph", + src_id: "a", + label: "KNOWS", + dst_id: "b", + properties: &[], + src_surrogate: Surrogate::new(1), + dst_surrogate: Surrogate::new(2), + }, + ); + assert_eq!(resp.status, Status::Ok); + + let wk = WriteKey { + vshard: vshard0(), + db: DatabaseId::DEFAULT, + tenant: TenantId::new(1), + collection: Box::from("graph"), + key: KeyRepr::Edge { + src: Box::from("a"), + label: Box::from("KNOWS"), + dst: Box::from("b"), + }, + }; + assert_eq!(core.write_index.key_version(&wk), Some(local(50))); + assert_eq!(core.watermark, Lsn::new(50)); + } + + #[test] + fn a_committed_transaction_records_its_write_versions() { + let (mut core, _, _, _dir) = make_core(); + let task = wal_task(60); + let plans = vec![PhysicalPlan::Document(DocumentOp::PointPut { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "batch"), + document_id: "d1".into(), + value: doc_value("a", "1"), + surrogate: Surrogate::new(11), + pk_bytes: Vec::new(), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + })]; + let resp = core.commit_plans_for_test(&task, 1, &plans, 60); + assert_eq!(resp.status, Status::Ok, "{:?}", resp.error_code); + + assert_eq!( + core.write_index.key_version(&surrogate_key("batch", 11)), + Some(local(60)) + ); + assert_eq!( + core.write_index.collection_version(&coll_key("batch")), + Some(local(60)) + ); + assert_eq!(core.watermark, Lsn::new(60)); + } + + #[test] + fn no_wal_lsn_records_nothing() { + let (mut core, _, _, _dir) = make_core(); + let task = ExecutionTask::new(make_request(PhysicalPlan::Document(DocumentOp::PointGet { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "x"), + document_id: "y".into(), + surrogate: None, + pk_bytes: Vec::new(), + rls_filters: Vec::new(), + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + }))); + point_put(&mut core, &task, "orders", "o1", 7, "1"); + assert_eq!( + core.write_index.key_version(&surrogate_key("orders", 7)), + None + ); + assert_eq!(core.watermark, Lsn::ZERO); + } + + #[test] + fn an_install_floor_bounds_the_installed_vshard_only() { + let (mut core, _, _, _dir) = make_core(); + point_put(&mut core, &wal_task(10), "orders", "o1", 7, "1"); + let cut = WriteVersion::logged(2, 300); + core.install_version_floor(&[ShardVersion { + vshard: 0, + version: cut, + }]); + assert_eq!(core.write_index.latest(vshard0()), cut); + assert_eq!( + core.write_index.key_version(&surrogate_key("orders", 7)), + None, + "the install replaced the row its version described" + ); + assert_eq!( + core.write_index.latest(VShardId::new(1)), + WriteVersion::ZERO + ); + } +} diff --git a/nodedb/src/data/executor/core_loop/write_index/record_stamps.rs b/nodedb/src/data/executor/core_loop/write_index/record_stamps.rs new file mode 100644 index 000000000..2977b25b3 --- /dev/null +++ b/nodedb/src/data/executor/core_loop/write_index/record_stamps.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The stamps of the WAL records the replay arms apply. +//! +//! A replay arm sees a record's payload and LSN, not its header. Restart +//! replay registers every record of the WAL here before the arms run, and a +//! committed redo install registers the one record it applies. A write an +//! arm applies then takes its record's vShard and the log position of the +//! data-group entry the record applies. +//! +//! A replicated write's records follow a `ChangePosition` marker. The marker +//! names the entry's position and the proposal key every record of the entry +//! carries in its header. Restart replay recovers each record's position by +//! that key, so a recovered write records the version its live apply did. + +use std::collections::HashMap; + +use nodedb_types::WriteVersion; +use nodedb_wal::WalRecord; +use nodedb_wal::record::RecordType; + +use crate::event::cdc::position::ChangePositionMarker; +use crate::types::{Lsn, VShardId}; + +use super::keys::WriteStamp; + +/// WAL record LSN to the stamp the record's writes take. +#[derive(Debug, Default)] +pub struct RecordStamps { + by_lsn: HashMap, + /// The highest version each vShard's registered records hold. + latest: HashMap, +} + +impl RecordStamps { + /// The stamps of `records`, a node's WAL. A record that follows the + /// `ChangePosition` marker of its proposal key takes the marker's entry + /// position. Every other record names no entry. + pub fn from_wal(records: &[WalRecord]) -> Self { + let mut ordered: Vec<&WalRecord> = records.iter().collect(); + ordered.sort_by_key(|record| record.header.lsn); + + let mut stamps = Self::default(); + let mut entry_of_key: HashMap = HashMap::new(); + for record in ordered { + if RecordType::from_raw(record.logical_record_type()) + == Some(RecordType::ChangePosition) + { + match ChangePositionMarker::from_bytes(&record.payload) { + Ok(marker) if marker.apply_key != 0 => { + entry_of_key.insert( + marker.apply_key, + WriteVersion::logged(marker.position.epoch, marker.position.log_index), + ); + } + Ok(_) => {} + Err(error) => tracing::warn!( + lsn = record.header.lsn, + %error, + "skipping an undecodable ChangePosition WAL marker" + ), + } + continue; + } + if record.header.vshard_id >= VShardId::COUNT { + continue; + } + let vshard = VShardId::new(record.header.vshard_id); + let key = record.apply_key(); + let entry = (key != 0) + .then(|| entry_of_key.get(&key).copied()) + .flatten(); + let latest = stamps.latest.entry(vshard).or_default(); + *latest = (*latest).max( + entry.unwrap_or_else(|| WriteVersion::local_after(*latest, record.header.lsn)), + ); + stamps.by_lsn.insert( + record.header.lsn, + WriteStamp { + vshard, + lsn: Lsn::new(record.header.lsn), + entry, + }, + ); + } + stamps + } + + /// Register the record a committed redo install applies. + pub fn insert(&mut self, stamp: WriteStamp) { + self.by_lsn.insert(stamp.lsn.as_u64(), stamp); + } + + /// Forget the record at `lsn`. + pub fn remove(&mut self, lsn: Lsn) { + self.by_lsn.remove(&lsn.as_u64()); + } + + /// Forget every record: restart replay finished. + pub fn clear(&mut self) { + self.by_lsn.clear(); + self.latest.clear(); + } + + /// The stamp of the record at `lsn`, if registered. + pub fn get(&self, lsn: u64) -> Option { + self.by_lsn.get(&lsn).copied() + } + + /// `stamp`, or the stamp of its record when the record is registered. + pub fn resolve(&self, stamp: WriteStamp) -> WriteStamp { + self.get(stamp.lsn.as_u64()).unwrap_or(stamp) + } + + /// The highest version each vShard's registered WAL records hold. + pub fn latest_by_vshard(&self) -> impl Iterator + '_ { + self.latest + .iter() + .map(|(vshard, version)| (*vshard, *version)) + } +} + +#[cfg(test)] +mod tests { + use nodedb_wal::record::WalRecordArgs; + + use super::*; + use crate::event::cdc::position::ReplicatedPosition; + + fn record(record_type: RecordType, lsn: u64, vshard: u32, apply_key: u64) -> WalRecord { + let payload = match record_type { + RecordType::ChangePosition => ChangePositionMarker { + apply_key, + position: ReplicatedPosition { + epoch: 3, + group_id: 7, + log_index: 100 + lsn, + }, + } + .to_bytes() + .to_vec(), + _ => vec![1], + }; + let header_key = match record_type { + RecordType::ChangePosition => 0, + _ => apply_key, + }; + WalRecord::new_stamped( + WalRecordArgs { + record_type: record_type as u32, + lsn, + tenant_id: 1, + vshard_id: vshard, + database_id: 0, + payload, + encryption_key: None, + preamble_bytes: None, + }, + nodedb_wal::record::RecordStamp { + apply_key: header_key, + event_source: nodedb_wal::NO_EVENT_SOURCE, + commit_hlc: 0, + }, + ) + .expect("build record") + } + + #[test] + fn a_record_after_its_marker_takes_the_entry_position() { + let records = vec![ + record(RecordType::ChangePosition, 10, 4, 0xAB), + record(RecordType::Put, 11, 4, 0xAB), + record(RecordType::Put, 12, 4, 0), + ]; + let stamps = RecordStamps::from_wal(&records); + + let replicated = stamps.get(11).expect("registered"); + assert_eq!(replicated.vshard, VShardId::new(4)); + assert_eq!(replicated.entry, Some(WriteVersion::logged(3, 110))); + + let local = stamps.get(12).expect("registered"); + assert_eq!(local.entry, None); + assert_eq!(stamps.get(10), None, "a marker writes no row"); + + let latest: HashMap = stamps.latest_by_vshard().collect(); + assert_eq!( + latest.get(&VShardId::new(4)), + Some(&WriteVersion::local_after(WriteVersion::logged(3, 110), 12)) + ); + } + + #[test] + fn resolve_keeps_an_unregistered_stamp() { + let stamps = RecordStamps::default(); + let stamp = WriteStamp { + vshard: VShardId::new(2), + lsn: Lsn::new(5), + entry: None, + }; + assert_eq!(stamps.resolve(stamp), stamp); + } +} diff --git a/nodedb/src/data/executor/core_loop/write_index/tests.rs b/nodedb/src/data/executor/core_loop/write_index/tests.rs new file mode 100644 index 000000000..ee7c8f336 --- /dev/null +++ b/nodedb/src/data/executor/core_loop/write_index/tests.rs @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `CoreLoop` fixtures shared by the write-version-index tests and by other +//! executor test modules (`make_core_with_dir` / `make_default_task` are +//! `pub` for exactly that reason). + +use std::time::{Duration, Instant}; + +use nodedb_bridge::buffer::RingBuffer; +use nodedb_physical::physical_plan::DocumentOp; +use nodedb_types::{DatabaseId, QualifiedCollection, Surrogate, TenantId, WriteVersion}; + +use crate::bridge::dispatch::{BridgeRequest, BridgeResponse}; +use crate::bridge::envelope::{PhysicalPlan, Priority, Request, Response}; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::point::put::PointPutExec; +use crate::data::executor::task::ExecutionTask; +use crate::types::{Lsn, ReadConsistency, RequestId, TraceId, VShardId}; + +pub(crate) fn make_core() -> ( + CoreLoop, + nodedb_bridge::buffer::Producer, + nodedb_bridge::buffer::Consumer, + tempfile::TempDir, +) { + let dir = tempfile::tempdir().unwrap(); + let (core, req_tx, resp_rx) = make_core_with_dir(dir.path()); + (core, req_tx, resp_rx, dir) +} + +pub fn make_core_with_dir( + dir: &std::path::Path, +) -> ( + CoreLoop, + nodedb_bridge::buffer::Producer, + nodedb_bridge::buffer::Consumer, +) { + let (req_tx, req_rx) = RingBuffer::channel::(64); + let (resp_tx, resp_rx) = RingBuffer::channel::(64); + let core = CoreLoop::open( + 0, + req_rx, + resp_tx, + dir, + std::sync::Arc::new(nodedb_types::OrdinalClock::new()), + crate::data::executor::core_loop::test_governor(), + ) + .unwrap(); + (core, req_tx, resp_rx) +} + +fn point_get() -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::PointGet { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "x"), + document_id: "y".into(), + surrogate: None, + pk_bytes: Vec::new(), + rls_filters: Vec::new(), + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + }) +} + +pub(crate) fn make_request(plan: PhysicalPlan) -> Request { + Request { + request_id: RequestId::new(1), + tenant_id: TenantId::new(1), + database_id: DatabaseId::DEFAULT, + vshard_id: VShardId::new(0), + plan, + deadline: Instant::now() + Duration::from_secs(5), + priority: Priority::Normal, + trace_id: TraceId::ZERO, + consistency: ReadConsistency::Strong, + idempotency_key: None, + event_source: crate::event::EventSource::User, + user_roles: Vec::new(), + user_id: None, + statement_digest: None, + txn_id: None, + wal_lsn: None, + resolved_now_ms: None, + commit_hlc: None, + entry_version: None, + admission: crate::bridge::envelope::Admission::Admitted, + } +} + +/// A minimal `ExecutionTask` (DEFAULT database/tenant, vShard 0, no WAL LSN) +/// for handler unit tests that only read `request.database_id`. The carried +/// plan is inert: edge/point handlers take their parameters directly. +pub fn make_default_task() -> ExecutionTask { + ExecutionTask::new(make_request(point_get())) +} + +/// A `{k: v}` document body: a standard MessagePack map. +pub(crate) fn doc_value(k: &str, v: &str) -> Vec { + let mut obj = std::collections::HashMap::new(); + obj.insert(k.to_string(), nodedb_types::Value::String(v.into())); + nodedb_types::value_to_msgpack(&nodedb_types::Value::Object(obj)).unwrap() +} + +/// An `ExecutionTask` carrying WAL LSN `lsn` and applying no data-group +/// entry: tenant 1, database DEFAULT, vShard 0. +pub(crate) fn wal_task(lsn: u64) -> ExecutionTask { + ExecutionTask::with_wal_lsn(make_request(point_get()), Some(Lsn::new(lsn))) +} + +/// An `ExecutionTask` carrying WAL LSN `lsn` that applies the data-group +/// entry at `entry`. +pub(crate) fn entry_task(lsn: u64, entry: WriteVersion) -> ExecutionTask { + ExecutionTask::with_wal_lsn( + Request { + entry_version: Some(entry), + ..make_request(point_get()) + }, + Some(Lsn::new(lsn)), + ) +} + +/// An `ExecutionTask` homing to `vshard_id`, carrying no WAL LSN. +pub(crate) fn task_with_vshard(vshard_id: VShardId) -> ExecutionTask { + ExecutionTask::new(Request { + vshard_id, + ..make_request(point_get()) + }) +} + +/// The version a write at `lsn` records on a vShard no entry ever wrote: +/// a single-node write. +pub(crate) fn local(lsn: u64) -> WriteVersion { + WriteVersion::local_after(WriteVersion::ZERO, lsn) +} + +/// The vShard a replayed write of `collection` in `db` records under when no +/// registered record stamp names one: the collection's home vShard. +pub(crate) fn replay_home(db: DatabaseId, collection: &str) -> VShardId { + nodedb_types::CollectionKey::from_qualified_str(db, collection) + .unwrap_or_else(|_| nodedb_types::CollectionKey::from_bare(db, collection)) + .vshard() +} + +/// Put `{a: value}` as document `document_id` of `collection` under `task`. +pub(crate) fn point_put( + core: &mut CoreLoop, + task: &ExecutionTask, + collection: &str, + document_id: &str, + surrogate: u32, + value: &str, +) -> Response { + core.execute_point_put( + task, + PointPutExec { + tid: 1, + collection, + document_id, + surrogate: Surrogate::new(surrogate), + value: &doc_value("a", value), + returning: None, + rls_filters: &[], + resolved_sum_targets: &[], + }, + ) +} diff --git a/nodedb/src/data/executor/core_loop/write_index/validate.rs b/nodedb/src/data/executor/core_loop/write_index/validate.rs new file mode 100644 index 000000000..51991a501 --- /dev/null +++ b/nodedb/src/data/executor/core_loop/write_index/validate.rs @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Calvin read-set validation against the per-core write versions. + +use nodedb_types::calvin::VersionedReadEntry; + +use crate::data::executor::task::ExecutionTask; +use crate::types::{TenantId, VShardId}; + +use super::super::CoreLoop; + +impl CoreLoop { + /// Whether this shard's slice of a transaction's versioned read-set is + /// still current against the local write versions. + /// + /// Filters the read-set to the entries homed on this request's vShard, + /// the only reads this core holds versions for, then checks each against + /// the write-version index. Short-circuits on the first entry that is no + /// longer current. An empty or fully-remote slice is vacuously current. + /// The `(database, tenant)` scope mirrors the write-version recorder so a + /// read validates against the same key space it was recorded in. + /// + /// Read versions are data-group log positions, so the verdict is the same + /// on every replica of the vShard, whichever node served the read. + /// + /// An entry's home is its `home_vshard` when set (a cross-shard graph read + /// names the key vShard it read edges on), else its collection's vShard, + /// by the same collection-in-database function the scheduler routes plans + /// with. A homed entry with no collection observed every collection on its + /// vShard, so it is current only while the vShard's latest version has + /// not passed its read version. + pub(in crate::data::executor) fn read_set_still_current( + &self, + task: &ExecutionTask, + tid: u64, + versioned_reads: &[VersionedReadEntry], + ) -> bool { + let db = task.request.database_id; + let tenant = TenantId::new(tid); + let local_vshard = task.request.vshard_id; + versioned_reads.iter().all(|entry| { + let home = match entry.home_vshard { + Some(home) if home < VShardId::COUNT => VShardId::new(home), + // A home past the vShard space names no shard: fail closed. + Some(_) => return false, + // An entry carries the plan's database-qualified name. One + // that does not de-qualify cannot be homed or validated, so + // the read set fails closed. + None => { + match nodedb_types::CollectionKey::from_qualified_str(db, &entry.collection) { + Ok(key) => key.vshard(), + Err(_) => return false, + } + } + }; + if home != local_vshard { + return true; + } + if entry.collection.is_empty() { + return self.write_index.latest(home) <= entry.read_version; + } + self.write_index.read_is_valid( + db, + tenant, + home, + &entry.collection, + &entry.key, + entry.read_version, + ) + }) + } +} + +#[cfg(test)] +mod tests { + use nodedb_physical::physical_plan::DocumentOp; + use nodedb_types::calvin::{EngineTag, ReadKeyIdent}; + use nodedb_types::{DatabaseId, QualifiedCollection, Surrogate, WriteVersion}; + + use super::super::keys::{KeyRepr, WriteStamp}; + use super::super::tests::{doc_value, make_core, make_request, task_with_vshard}; + use super::*; + use crate::bridge::envelope::{PhysicalPlan, Status}; + use crate::data::executor::task::ExecutionTask; + use crate::types::Lsn; + + /// The vShard `collection` homes to in the default database: mirrors the + /// homing `read_set_still_current` filters entries by. + fn local_vshard(collection: &str) -> VShardId { + nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, collection).vshard() + } + + /// Some vShard other than `than`, for exercising the cross-shard filter. + fn other_vshard(than: VShardId) -> VShardId { + VShardId::new((than.as_u32() + 1) % VShardId::COUNT) + } + + fn at(index: u64) -> WriteVersion { + WriteVersion::logged(0, index) + } + + /// Record a write applying the entry at `index` on `vshard`. + fn write( + core: &mut CoreLoop, + vshard: VShardId, + collection: &str, + key: Option, + index: u64, + ) { + core.note_write( + DatabaseId::DEFAULT, + TenantId::new(1), + collection, + key, + WriteStamp { + vshard, + lsn: Lsn::new(index), + entry: Some(at(index)), + }, + ); + } + + fn point_entry(collection: &str, surrogate: u32, read: u64) -> VersionedReadEntry { + VersionedReadEntry { + engine: EngineTag::Document, + collection: collection.to_string(), + key: ReadKeyIdent::Point(KeyRepr::Surrogate(surrogate)), + read_version: at(read), + home_vshard: None, + } + } + + fn predicate_entry(collection: &str, read: u64) -> VersionedReadEntry { + VersionedReadEntry { + key: ReadKeyIdent::Predicate, + ..point_entry(collection, 0, read) + } + } + + fn homed_entry(collection: &str, home: VShardId, read: u64) -> VersionedReadEntry { + VersionedReadEntry { + home_vshard: Some(home.as_u32()), + ..predicate_entry(collection, read) + } + } + + #[test] + fn a_stale_point_read_is_not_current_and_a_fresh_one_is() { + let (mut core, _, _, _dir) = make_core(); + let vshard = local_vshard("orders"); + write(&mut core, vshard, "orders", Some(KeyRepr::Surrogate(7)), 20); + let task = task_with_vshard(vshard); + assert!(!core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 10)])); + assert!(core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 20)])); + assert!(core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 30)])); + } + + /// The read was served on another replica of the vShard. Its version is + /// the log position that replica had applied, which this replica records + /// for the same writes, so the read validates here by version alone. + #[test] + fn a_read_served_on_another_replica_validates_by_its_log_position() { + let vshard = local_vshard("orders"); + let (mut serving, _, _, _serving_dir) = make_core(); + let (mut validating, _, _, _validating_dir) = make_core(); + // Both replicas apply the entry at index 20, at different WAL LSNs. + for (core, lsn) in [(&mut serving, 7_000u64), (&mut validating, 31u64)] { + core.note_write( + DatabaseId::DEFAULT, + TenantId::new(1), + "orders", + Some(KeyRepr::Surrogate(7)), + WriteStamp { + vshard, + lsn: Lsn::new(lsn), + entry: Some(at(20)), + }, + ); + } + let read_version = serving + .write_index + .key_version(&super::super::keys::WriteKey { + vshard, + db: DatabaseId::DEFAULT, + tenant: TenantId::new(1), + collection: Box::from("orders"), + key: KeyRepr::Surrogate(7), + }) + .expect("serving replica recorded the write"); + let read = VersionedReadEntry { + read_version, + ..point_entry("orders", 7, 0) + }; + let task = task_with_vshard(vshard); + assert!(validating.read_set_still_current(&task, 1, std::slice::from_ref(&read))); + + // A conflicting write the validating replica applies after the read + // makes the same read stale. + write( + &mut validating, + vshard, + "orders", + Some(KeyRepr::Surrogate(7)), + 21, + ); + assert!(!validating.read_set_still_current(&task, 1, &[read])); + } + + /// A read that found no row answers `NotFound` with the version it + /// observed. That version, carried to the coordinator on the wire, keeps + /// the miss valid until a write of the missed key lands after it. + #[test] + fn a_missed_read_carries_its_version_and_stays_valid_until_the_key_is_written() { + let (mut core, _, _, _dir) = make_core(); + let vshard = local_vshard("orders"); + write(&mut core, vshard, "orders", Some(KeyRepr::Surrogate(7)), 20); + let read = ExecutionTask::new(make_request(PhysicalPlan::Document(DocumentOp::PointGet { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "orders"), + document_id: "missing".into(), + surrogate: Some(Surrogate::new(8)), + pk_bytes: Vec::new(), + rls_filters: Vec::new(), + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + }))); + let miss = core.response_error(&read, crate::bridge::envelope::ErrorCode::NotFound); + assert_eq!(miss.status, Status::Error); + let observed = miss + .read_versions + .of(vshard) + .expect("a miss reports the version it observed"); + assert_eq!(observed, at(20)); + + let entry = VersionedReadEntry { + read_version: observed, + ..point_entry("orders", 8, 0) + }; + let task = task_with_vshard(vshard); + assert!(core.read_set_still_current(&task, 1, std::slice::from_ref(&entry))); + write(&mut core, vshard, "orders", Some(KeyRepr::Surrogate(8)), 21); + assert!(!core.read_set_still_current(&task, 1, &[entry])); + } + + fn array_slice(vshard: VShardId) -> ExecutionTask { + ExecutionTask::new(crate::bridge::envelope::Request { + vshard_id: vshard, + ..make_request(PhysicalPlan::Array( + nodedb_physical::physical_plan::ArrayOp::Slice { + array_id: nodedb_array::types::ArrayId::in_database( + TenantId::new(1), + DatabaseId::DEFAULT, + "grid", + ), + slice_msgpack: Vec::new(), + attr_projection: Vec::new(), + limit: 0, + cell_filter: None, + hilbert_range: None, + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + }, + )) + }) + } + + /// An array read reports the vShard it covers even when no write of it + /// reached this core. A first write of that vShard then makes the read + /// stale: no tile can appear after the read unseen. + #[test] + fn an_array_read_covers_a_vshard_no_write_reached_and_a_first_write_makes_it_stale() { + let (mut core, _, _, _dir) = make_core(); + let vshard = local_vshard("grid"); + let observed = core + .read_versions(&array_slice(vshard)) + .of(vshard) + .expect("an array read reports its vShard with no write recorded"); + assert_eq!(observed, WriteVersion::ZERO); + + let entry = VersionedReadEntry { + engine: EngineTag::Array, + read_version: observed, + ..predicate_entry("grid", 0) + }; + let task = task_with_vshard(vshard); + assert!(core.read_set_still_current(&task, 1, std::slice::from_ref(&entry))); + write(&mut core, vshard, "grid", None, 5); + assert!(!core.read_set_still_current(&task, 1, &[entry])); + } + + /// An installed vShard with no recorded array write reports its install + /// bound, so the read stays current until a write lands above it. + #[test] + fn an_array_read_of_an_installed_vshard_reports_the_install_bound() { + let (mut core, _, _, _dir) = make_core(); + let vshard = local_vshard("grid"); + core.install_version_floor(&[nodedb_types::ShardVersion { + vshard: vshard.as_u32(), + version: at(9), + }]); + let observed = core + .read_versions(&array_slice(vshard)) + .of(vshard) + .expect("an array read reports its vShard"); + assert_eq!(observed, at(9)); + } + + #[test] + fn read_entry_homing_to_a_different_vshard_is_filtered_out() { + let (mut core, _, _, _dir) = make_core(); + let local = local_vshard("orders"); + write(&mut core, local, "orders", Some(KeyRepr::Surrogate(7)), 20); + let remote_task = task_with_vshard(other_vshard(local)); + assert!(core.read_set_still_current(&remote_task, 1, &[point_entry("orders", 7, 10)])); + } + + #[test] + fn a_stale_predicate_read_is_not_current_and_a_fresh_one_is() { + let (mut core, _, _, _dir) = make_core(); + let vshard = local_vshard("orders"); + write(&mut core, vshard, "orders", None, 20); + let task = task_with_vshard(vshard); + assert!(!core.read_set_still_current(&task, 1, &[predicate_entry("orders", 10)])); + assert!(core.read_set_still_current(&task, 1, &[predicate_entry("orders", 20)])); + } + + #[test] + fn a_homed_read_validates_on_its_home_not_its_collection_vshard() { + let (mut core, _, _, _dir) = make_core(); + let home = other_vshard(local_vshard("edges")); + write(&mut core, home, "edges", None, 20); + let stale = vec![homed_entry("edges", home, 10)]; + assert!(!core.read_set_still_current(&task_with_vshard(home), 1, &stale)); + assert!( + core.read_set_still_current(&task_with_vshard(local_vshard("edges")), 1, &stale), + "the collection's own vShard does not hold a homed read" + ); + let fresh = vec![homed_entry("edges", home, 20)]; + assert!(core.read_set_still_current(&task_with_vshard(home), 1, &fresh)); + } + + #[test] + fn a_homed_read_of_every_collection_validates_against_the_vshard_latest_version() { + let (mut core, _, _, _dir) = make_core(); + let home = local_vshard("edges"); + write(&mut core, home, "unrelated", None, 20); + // A write to another vShard on the same core never moves the home. + write(&mut core, other_vshard(home), "unrelated", None, 900); + let task = task_with_vshard(home); + assert!(!core.read_set_still_current(&task, 1, &[homed_entry("", home, 10)])); + assert!(core.read_set_still_current(&task, 1, &[homed_entry("", home, 20)])); + } + + #[test] + fn empty_read_set_is_vacuously_current() { + let (core, _, _, _dir) = make_core(); + let task = task_with_vshard(VShardId::new(0)); + assert!(core.read_set_still_current(&task, 1, &[])); + } + + #[test] + fn a_read_that_predates_a_committed_write_is_no_longer_current() { + let (mut core, _, _, _dir) = make_core(); + let vshard = local_vshard("orders"); + let entry = at(10); + let write_task = ExecutionTask::with_wal_lsn( + crate::bridge::envelope::Request { + entry_version: Some(entry), + ..task_with_vshard(vshard).request + }, + Some(Lsn::new(77)), + ); + let write_plans = vec![PhysicalPlan::Document(DocumentOp::PointPut { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "orders"), + document_id: "o7".into(), + value: doc_value("a", "1"), + surrogate: Surrogate::new(7), + pk_bytes: Vec::new(), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + })]; + let write_resp = core.commit_plans_for_test(&write_task, 1, &write_plans, 77); + assert_eq!(write_resp.status, Status::Ok, "{:?}", write_resp.error_code); + + let task = task_with_vshard(vshard); + assert!(!core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 5)])); + assert!(core.read_set_still_current(&task, 1, &[point_entry("orders", 7, 10)])); + } +} diff --git a/nodedb/src/data/executor/deadline.rs b/nodedb/src/data/executor/deadline.rs index 19475922c..1262cf82c 100644 --- a/nodedb/src/data/executor/deadline.rs +++ b/nodedb/src/data/executor/deadline.rs @@ -172,6 +172,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission, }) } diff --git a/nodedb/src/data/executor/dispatch/array/aggregate.rs b/nodedb/src/data/executor/dispatch/array/aggregate.rs index 967ce26b6..bd0a03648 100644 --- a/nodedb/src/data/executor/dispatch/array/aggregate.rs +++ b/nodedb/src/data/executor/dispatch/array/aggregate.rs @@ -401,6 +401,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/src/data/executor/dispatch/array/elementwise.rs b/nodedb/src/data/executor/dispatch/array/elementwise.rs index 1829389d1..844268f72 100644 --- a/nodedb/src/data/executor/dispatch/array/elementwise.rs +++ b/nodedb/src/data/executor/dispatch/array/elementwise.rs @@ -332,6 +332,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/src/data/executor/dispatch/array/mutate.rs b/nodedb/src/data/executor/dispatch/array/mutate.rs index ce5be1428..e66ad9ef0 100644 --- a/nodedb/src/data/executor/dispatch/array/mutate.rs +++ b/nodedb/src/data/executor/dispatch/array/mutate.rs @@ -184,12 +184,13 @@ impl CoreLoop { ) -> crate::Result<()> { if wal_lsn > 0 { let lsn = crate::types::Lsn::new(wal_lsn); - self.note_write_lsn( + let stamp = self.task_write_stamp_at(task, lsn); + self.note_write( task.request.database_id, task.request.tenant_id, &array_id.name, None, - lsn, + stamp, ); self.floors.applied_prefix.note_applied(lsn); } @@ -330,6 +331,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/src/data/executor/dispatch/array/read.rs b/nodedb/src/data/executor/dispatch/array/read.rs index adb5746c9..387469017 100644 --- a/nodedb/src/data/executor/dispatch/array/read.rs +++ b/nodedb/src/data/executor/dispatch/array/read.rs @@ -533,6 +533,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/src/data/executor/dispatch/array/surrogate_scan.rs b/nodedb/src/data/executor/dispatch/array/surrogate_scan.rs index 6f0128a90..afc225c9a 100644 --- a/nodedb/src/data/executor/dispatch/array/surrogate_scan.rs +++ b/nodedb/src/data/executor/dispatch/array/surrogate_scan.rs @@ -173,6 +173,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/src/data/executor/dispatch/bitmap/hashjoin_inline.rs b/nodedb/src/data/executor/dispatch/bitmap/hashjoin_inline.rs index 33ff4b011..6c384b9e5 100644 --- a/nodedb/src/data/executor/dispatch/bitmap/hashjoin_inline.rs +++ b/nodedb/src/data/executor/dispatch/bitmap/hashjoin_inline.rs @@ -153,6 +153,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/dispatch/graph/node_labels.rs b/nodedb/src/data/executor/dispatch/graph/node_labels.rs index d4840843e..659bce946 100644 --- a/nodedb/src/data/executor/dispatch/graph/node_labels.rs +++ b/nodedb/src/data/executor/dispatch/graph/node_labels.rs @@ -163,6 +163,7 @@ mod tests { wal_lsn: Some(Lsn::new(lsn)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/dispatch/meta.rs b/nodedb/src/data/executor/dispatch/meta.rs index ae3657c19..3fb037953 100644 --- a/nodedb/src/data/executor/dispatch/meta.rs +++ b/nodedb/src/data/executor/dispatch/meta.rs @@ -75,14 +75,23 @@ impl CoreLoop { replace_mode, collections_to_clear, group_vshards, - } => self.execute_restore_tenant_snapshot( - task, - *tenant_id, - snapshot, - *replace_mode, - collections_to_clear, - group_vshards, - ), + version_floor, + } => { + let response = self.execute_restore_tenant_snapshot( + task, + *tenant_id, + snapshot, + *replace_mode, + collections_to_clear, + group_vshards, + ); + // The installed rows carry no versions: each vShard holds + // every write through the snapshot's cut. + if response.status == crate::bridge::envelope::Status::Ok { + self.install_version_floor(version_floor); + } + response + } MetaOp::ConvertCollection { collection, @@ -395,6 +404,7 @@ mod txn_created_columnar_engine_tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/dispatch/timeseries.rs b/nodedb/src/data/executor/dispatch/timeseries.rs index e0f305f09..56b7aaf8d 100644 --- a/nodedb/src/data/executor/dispatch/timeseries.rs +++ b/nodedb/src/data/executor/dispatch/timeseries.rs @@ -244,6 +244,7 @@ mod tests { wal_lsn: envelope_lsn.map(Lsn::new), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Admitted, }) } diff --git a/nodedb/src/data/executor/graph_label_checkpoint/mod.rs b/nodedb/src/data/executor/graph_label_checkpoint/mod.rs index 8be4f5996..7c0052790 100644 --- a/nodedb/src/data/executor/graph_label_checkpoint/mod.rs +++ b/nodedb/src/data/executor/graph_label_checkpoint/mod.rs @@ -17,7 +17,7 @@ //! durable trace a label ever existed, which is exactly why //! `wal_replay_graph_labels.rs` is a standalone replay pass. Yet a label write //! advances the core watermark (`GraphOp::SetNodeLabels` goes through -//! `note_write_lsn` like any other write), so the periodic checkpoint reported +//! `note_write` like any other write), so the periodic checkpoint reported //! it as durable and the manager truncated the segments holding the only copy. //! The label silently vanished on the next restart, while the edges around it //! came back intact — the node was still there, `MATCH (a:Person)` just stopped diff --git a/nodedb/src/data/executor/handlers/aggregate/exec.rs b/nodedb/src/data/executor/handlers/aggregate/exec.rs index cb05daed7..a6174469d 100644 --- a/nodedb/src/data/executor/handlers/aggregate/exec.rs +++ b/nodedb/src/data/executor/handlers/aggregate/exec.rs @@ -456,6 +456,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs b/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs index 0b9f625bc..fd94e30ff 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs @@ -184,18 +184,18 @@ impl CoreLoop { ); // Record the committed delete's write version against its // surrogate + collection. - self.note_surrogate_write_lsn(task, tid, collection, row_surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, row_surrogate.as_u32()); // Record the removed secondary-index tuples into the // per-index write-value substrate, recomputed from the // pre-delete document (see `index_paths` comment above). - if let (Some(lsn), Some(doc)) = (task.wal_lsn(), pre_delete_doc.as_ref()) { + if let (Some(stamp), Some(doc)) = (self.task_write_stamp(task), pre_delete_doc.as_ref()) { let tuples = self.index_tuples_for_doc(doc, index_paths); self.note_index_write_values( task.request.database_id, crate::types::TenantId::new(tid), collection, &tuples, - lsn, + stamp, ); } // The removal, journalled after apply: the plan carries no diff --git a/nodedb/src/data/executor/handlers/bulk_dml/update.rs b/nodedb/src/data/executor/handlers/bulk_dml/update.rs index a2b5a9b20..52cf20d79 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/update.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/update.rs @@ -324,13 +324,13 @@ impl CoreLoop { // Published only after the commit succeeded — the same // ordering the reindex helper used when it owned the // transaction. - if let Some(lsn) = task.wal_lsn() { + if let Some(stamp) = self.task_write_stamp(task) { self.note_index_write_values( task.request.database_id, crate::types::TenantId::new(tid), collection, &touched, - lsn, + stamp, ); } self.doc_cache.put( @@ -343,7 +343,7 @@ impl CoreLoop { // Record the committed row's write version against its // surrogate + collection. let surrogate = storage_key.surrogate(); - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); // Re-index the row's vectors from the new body (soft-delete the // old HNSW node + insert the new one, keyed by the stable // surrogate). No-op unless the collection has a vector field diff --git a/nodedb/src/data/executor/handlers/columnar_mutation.rs b/nodedb/src/data/executor/handlers/columnar_mutation.rs index 72f240929..7f4a214a3 100644 --- a/nodedb/src/data/executor/handlers/columnar_mutation.rs +++ b/nodedb/src/data/executor/handlers/columnar_mutation.rs @@ -143,7 +143,7 @@ impl CoreLoop { // re-executing it against the identical restored state matches nothing // again. if affected > 0 { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } debug!(core = self.core_id, %collection, affected, "columnar update complete"); @@ -250,7 +250,7 @@ impl CoreLoop { // it, needlessly holding WAL segments and denying a concurrent // transaction the conflict it should see against the rows this removed. if affected > 0 { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } debug!(core = self.core_id, %collection, affected, "columnar delete complete"); diff --git a/nodedb/src/data/executor/handlers/columnar_read/materialize_scan.rs b/nodedb/src/data/executor/handlers/columnar_read/materialize_scan.rs index 84f1552e0..c6f3bddef 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/materialize_scan.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/materialize_scan.rs @@ -444,6 +444,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs b/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs index 88a7c64e9..17dff10bb 100644 --- a/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs +++ b/nodedb/src/data/executor/handlers/columnar_read/scan/execute.rs @@ -468,6 +468,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/handlers/columnar_resolved_mutation.rs b/nodedb/src/data/executor/handlers/columnar_resolved_mutation.rs index aa0d280b1..f7ff1b524 100644 --- a/nodedb/src/data/executor/handlers/columnar_resolved_mutation.rs +++ b/nodedb/src/data/executor/handlers/columnar_resolved_mutation.rs @@ -145,7 +145,7 @@ impl CoreLoop { // a checkpoint stamp that already contains it, and replay would // duplicate the row. if affected > 0 { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } debug!(core = self.core_id, %collection, affected, "columnar resolved update complete"); @@ -224,7 +224,7 @@ impl CoreLoop { } if affected > 0 { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } debug!(core = self.core_id, %collection, affected, "columnar resolved delete complete"); diff --git a/nodedb/src/data/executor/handlers/columnar_truncate.rs b/nodedb/src/data/executor/handlers/columnar_truncate.rs index cdf166124..51c4bc833 100644 --- a/nodedb/src/data/executor/handlers/columnar_truncate.rs +++ b/nodedb/src/data/executor/handlers/columnar_truncate.rs @@ -78,7 +78,7 @@ impl CoreLoop { self.checkpoint_coordinator .mark_dirty("columnar", truncated); - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); self.invalidate_aggregate_cache_for_collection(db.as_u64(), tid.as_u64(), collection); debug!(core = self.core_id, %collection, truncated, "columnar truncate complete"); diff --git a/nodedb/src/data/executor/handlers/columnar_write/insert.rs b/nodedb/src/data/executor/handlers/columnar_write/insert.rs index c6d7f67bb..b992b8539 100644 --- a/nodedb/src/data/executor/handlers/columnar_write/insert.rs +++ b/nodedb/src/data/executor/handlers/columnar_write/insert.rs @@ -201,7 +201,7 @@ impl CoreLoop { // Advance the collection floor for this committed columnar write. if accepted > 0 { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } // On the sync path, advance the HWM and return SyncAckResult payload. @@ -244,7 +244,7 @@ impl CoreLoop { watermark_lsn: self.watermark, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs b/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs index b972645e6..a59701e9d 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs @@ -85,7 +85,7 @@ impl CoreLoop { /// The one divergence from the static path: OLLP predicate verification /// (leader-only) runs HERE, before staging, via /// [`CoreLoop::verify_calvin_active_ollp`]. The dependent-read path has no - /// LSN-versioned read-set to vote on; its conflict detector is the OLLP + /// write-versioned read-set to vote on; its conflict detector is the OLLP /// `actual != predicted` re-check. A mismatch returns `OllpRetryRequired` /// and stages nothing, so no redo entry exists for it. The Control /// Plane scheduler releases locks and re-recons on `OllpRetryRequired`. @@ -199,7 +199,7 @@ impl CoreLoop { // The dependent-read path carries no versioned read-set. Its OLLP // check passed above, so the slice votes commit. stage_vote: Some(StageVote::Commit), - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/data/executor/handlers/control/calvin/shared.rs b/nodedb/src/data/executor/handlers/control/calvin/shared.rs index 45448cf02..540f694e6 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/shared.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/shared.rs @@ -98,6 +98,7 @@ pub(in crate::data::executor) mod test_support { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }; ExecutionTask::new(request) diff --git a/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs b/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs index 706be5e9d..96f740035 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs @@ -25,7 +25,7 @@ impl CoreLoop { /// Validate a static-set Calvin transaction and stage it for commit. /// /// Computes the local commit vote by checking whether this participant's - /// slice of the transaction's LSN-versioned read-set is still current + /// slice of the transaction's versioned read-set is still current /// against the per-core write versions, then STAGES the write plans into /// the synthetic overlay and the commit-pending buffer keyed by /// `(epoch, position)`. It performs NO base mutation and fires NO side @@ -149,7 +149,7 @@ impl CoreLoop { watermark_lsn: self.watermark, error_code: None, stage_vote: Some(vote), - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } @@ -241,29 +241,39 @@ mod tests { ); } - /// A predicate read of `collection` on vShard 0, taken at `read_lsn`. - fn homed_read(collection: &str, read_lsn: u64) -> VersionedReadEntry { + /// A predicate read of `collection` on vShard 0, taken after the entry at + /// `index`. + fn homed_read(collection: &str, index: u64) -> VersionedReadEntry { VersionedReadEntry { engine: nodedb_types::calvin::EngineTag::Document, collection: collection.to_string(), key: nodedb_types::calvin::ReadKeyIdent::Predicate, - read_lsn: crate::types::Lsn::new(read_lsn), + read_version: nodedb_types::WriteVersion::logged(0, index), home_vshard: Some(0), - served_by: 0, } } - #[test] - fn static_stage_with_a_superseded_read_votes_serialization_conflict() { - let dir = tempfile::tempdir().unwrap(); - let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - core.note_write_lsn( + /// Record a write of `collection` applying the entry at `index` on + /// vShard 0. + fn write_at(core: &mut CoreLoop, collection: &str, index: u64) { + core.note_write( DatabaseId::DEFAULT, TenantId::new(1), - "orders", + collection, None, - crate::types::Lsn::new(20), + crate::data::executor::core_loop::write_index::WriteStamp { + vshard: crate::types::VShardId::new(0), + lsn: crate::types::Lsn::new(index), + entry: Some(nodedb_types::WriteVersion::logged(0, index)), + }, ); + } + + #[test] + fn static_stage_with_a_superseded_read_votes_serialization_conflict() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + write_at(&mut core, "orders", 20); let task = make_task(); let tenant_id = TenantId::new(1); let ctx = CalvinExecCtx { @@ -292,13 +302,7 @@ mod tests { fn static_stage_with_a_current_read_votes_commit() { let dir = tempfile::tempdir().unwrap(); let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); - core.note_write_lsn( - DatabaseId::DEFAULT, - TenantId::new(1), - "orders", - None, - crate::types::Lsn::new(20), - ); + write_at(&mut core, "orders", 20); let task = make_task(); let tenant_id = TenantId::new(1); let ctx = CalvinExecCtx { diff --git a/nodedb/src/data/executor/handlers/control/calvin_resolve.rs b/nodedb/src/data/executor/handlers/control/calvin_resolve.rs index 991fcef34..52c16f3f2 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_resolve.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_resolve.rs @@ -173,6 +173,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }; ExecutionTask::new(request) diff --git a/nodedb/src/data/executor/handlers/control/crdt.rs b/nodedb/src/data/executor/handlers/control/crdt.rs index 7a05e657e..e7a0beabf 100644 --- a/nodedb/src/data/executor/handlers/control/crdt.rs +++ b/nodedb/src/data/executor/handlers/control/crdt.rs @@ -183,7 +183,7 @@ impl CoreLoop { ) { crate::engine::crdt::tenant_state::ValidatedApplyOutcome::Clean { .. } => { self.checkpoint_coordinator.mark_dirty("crdt", 1); - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); self.response_ok(task) } crate::engine::crdt::tenant_state::ValidatedApplyOutcome::Rejected(reason) => { diff --git a/nodedb/src/data/executor/handlers/control/crdt_apply/gated.rs b/nodedb/src/data/executor/handlers/control/crdt_apply/gated.rs index e7113d0ea..c9f86ef4e 100644 --- a/nodedb/src/data/executor/handlers/control/crdt_apply/gated.rs +++ b/nodedb/src/data/executor/handlers/control/crdt_apply/gated.rs @@ -407,7 +407,7 @@ impl CoreLoop { ); } if imported_authoritative { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } // The high-water-mark and the client-visible frame are decided diff --git a/nodedb/src/data/executor/handlers/control/crdt_apply/local.rs b/nodedb/src/data/executor/handlers/control/crdt_apply/local.rs index 68eaab503..2e3c15ea0 100644 --- a/nodedb/src/data/executor/handlers/control/crdt_apply/local.rs +++ b/nodedb/src/data/executor/handlers/control/crdt_apply/local.rs @@ -162,13 +162,13 @@ impl CoreLoop { &bytes, ); if imported_authoritative { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } } Ok(None) if imported_authoritative => { // A headless import has no sparse projection, but still // changed authoritative Loro state. - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } Ok(None) => {} Err(refusal) => { diff --git a/nodedb/src/data/executor/handlers/control/crdt_preview.rs b/nodedb/src/data/executor/handlers/control/crdt_preview.rs index 12568e53b..b02e8d6ef 100644 --- a/nodedb/src/data/executor/handlers/control/crdt_preview.rs +++ b/nodedb/src/data/executor/handlers/control/crdt_preview.rs @@ -208,7 +208,13 @@ mod tests { ); let import = core.execute_crdt_import_snapshot(&task, 1, "docs", &snapshot); assert_eq!(import.status, Status::Ok); - assert_eq!(import.read_version_lsn, crate::types::Lsn::new(73)); + assert_eq!( + import.read_versions.of(task.request.vshard_id), + Some(nodedb_types::WriteVersion::local_after( + nodedb_types::WriteVersion::ZERO, + 73 + )) + ); assert_eq!(import.watermark_lsn, crate::types::Lsn::new(73)); let before = core .execute_crdt_read(&task, "docs", "one") @@ -455,7 +461,13 @@ mod tests { }, ); assert_eq!(applied.status, Status::Ok); - assert_eq!(applied.read_version_lsn, crate::types::Lsn::new(91)); + assert_eq!( + applied.read_versions.of(task.request.vshard_id), + Some(nodedb_types::WriteVersion::local_after( + nodedb_types::WriteVersion::ZERO, + 91 + )) + ); assert_eq!(applied.watermark_lsn, crate::types::Lsn::new(91)); assert!( core.crdt_engines diff --git a/nodedb/src/data/executor/handlers/document/apply_balance_delta.rs b/nodedb/src/data/executor/handlers/document/apply_balance_delta.rs index e35c43b4f..7922753c7 100644 --- a/nodedb/src/data/executor/handlers/document/apply_balance_delta.rs +++ b/nodedb/src/data/executor/handlers/document/apply_balance_delta.rs @@ -143,7 +143,7 @@ impl CoreLoop { ); } - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); self.checkpoint_coordinator.mark_dirty("sparse", 1); let mut response = self.response_affected(task, 1); diff --git a/nodedb/src/data/executor/handlers/document/resolve/apply_row.rs b/nodedb/src/data/executor/handlers/document/resolve/apply_row.rs index 102f17d4d..a00e1785f 100644 --- a/nodedb/src/data/executor/handlers/document/resolve/apply_row.rs +++ b/nodedb/src/data/executor/handlers/document/resolve/apply_row.rs @@ -205,7 +205,7 @@ impl CoreLoop { self.checkpoint_coordinator.mark_dirty("sparse", 1); // Same post-commit index bookkeeping `execute_point_update` runs. - if let Some(lsn) = task.wal_lsn() { + if let Some(stamp) = self.task_write_stamp(task) { let mut tuples = std::mem::take(&mut outcome.secondary_index_added); tuples.append(&mut outcome.secondary_index_removed); tuples.append(&mut outcome.bitemporal_index_tuples); @@ -214,7 +214,7 @@ impl CoreLoop { crate::types::TenantId::new(tid), collection, &tuples, - lsn, + stamp, ); } @@ -227,7 +227,7 @@ impl CoreLoop { &stored_bytes, precondition, ); - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); // The row, journalled after apply from the body `apply_point_put` // took. A resolved write's mutations can name several collections, @@ -347,8 +347,8 @@ impl CoreLoop { self.checkpoint_coordinator.mark_dirty("sparse", 1); if let Some(prior_bytes) = outcome.prior_value.as_deref() { - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); - if let Some(lsn) = task.wal_lsn() { + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); + if let Some(stamp) = self.task_write_stamp(task) { let mut tuples = outcome.secondary_index_tuples; tuples.extend(outcome.bitemporal_index_tuples); self.note_index_write_values( @@ -356,7 +356,7 @@ impl CoreLoop { crate::types::TenantId::new(tid), collection, &tuples, - lsn, + stamp, ); } self.emit_document_delete_event( diff --git a/nodedb/src/data/executor/handlers/document/write/batch_insert.rs b/nodedb/src/data/executor/handlers/document/write/batch_insert.rs index 873e56b93..04aa5dd0d 100644 --- a/nodedb/src/data/executor/handlers/document/write/batch_insert.rs +++ b/nodedb/src/data/executor/handlers/document/write/batch_insert.rs @@ -358,17 +358,23 @@ impl CoreLoop { return self.response_error(task, e); } + // Record each committed row's version against its surrogate and + // collection. + for (_, key) in &applied { + self.note_surrogate_write(task, tid, collection, key.surrogate().as_u32()); + } + // Record each committed row's touched secondary-index values into the // per-index write-value substrate. This runs only after the batch has // durably committed. - if let Some(lsn) = task.wal_lsn() { + if let Some(stamp) = self.task_write_stamp(task) { for tuples in &row_index_tuples { self.note_index_write_values( task.request.database_id, crate::types::TenantId::new(tid), collection, tuples, - lsn, + stamp, ); } } @@ -501,6 +507,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, }) } diff --git a/nodedb/src/data/executor/handlers/fts_sync.rs b/nodedb/src/data/executor/handlers/fts_sync.rs index 927cb84b7..a693eb1f1 100644 --- a/nodedb/src/data/executor/handlers/fts_sync.rs +++ b/nodedb/src/data/executor/handlers/fts_sync.rs @@ -69,7 +69,7 @@ impl CoreLoop { { Ok(()) => { // Advance the collection floor for this committed FTS write. - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); if let Some(prov) = provenance { self.sync_commit(prov); return self.sync_ack_response(task, AckStatus::Applied, prov.seq); @@ -133,7 +133,7 @@ impl CoreLoop { { Ok(()) => { // Advance the collection floor for this committed FTS delete. - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); if let Some(prov) = provenance { self.sync_commit(prov); return self.sync_ack_response(task, AckStatus::Applied, prov.seq); diff --git a/nodedb/src/data/executor/handlers/graph_edge_write/delete.rs b/nodedb/src/data/executor/handlers/graph_edge_write/delete.rs index 36330f675..337316276 100644 --- a/nodedb/src/data/executor/handlers/graph_edge_write/delete.rs +++ b/nodedb/src/data/executor/handlers/graph_edge_write/delete.rs @@ -145,8 +145,8 @@ impl CoreLoop { undo.push(UndoEntry::EdgeWrite(Box::new(edge_undo))); } self.checkpoint_coordinator.mark_dirty("sparse", 1); - self.note_edge_write_lsn(task, tid, collection, src_id, label, dst_id); - // CDC: emit after `note_edge_write_lsn` so the event LSN matches + self.note_edge_write(task, tid, collection, src_id, label, dst_id); + // CDC: emit after `note_edge_write` so the event LSN matches // this edge's WAL LSN (the WAL-replay reconstruction key). self.emit_graph_edge_event( task, diff --git a/nodedb/src/data/executor/handlers/graph_edge_write/delete_batch.rs b/nodedb/src/data/executor/handlers/graph_edge_write/delete_batch.rs index 3a18c67ae..b533a3101 100644 --- a/nodedb/src/data/executor/handlers/graph_edge_write/delete_batch.rs +++ b/nodedb/src/data/executor/handlers/graph_edge_write/delete_batch.rs @@ -144,7 +144,7 @@ impl CoreLoop { .mark_dirty("sparse", edges.len()); } for edge in edges { - self.note_edge_write_lsn( + self.note_edge_write( task, tid, edge.collection.as_str(), diff --git a/nodedb/src/data/executor/handlers/graph_edge_write/put.rs b/nodedb/src/data/executor/handlers/graph_edge_write/put.rs index 9fadb68bc..d4ba40f4c 100644 --- a/nodedb/src/data/executor/handlers/graph_edge_write/put.rs +++ b/nodedb/src/data/executor/handlers/graph_edge_write/put.rs @@ -152,8 +152,8 @@ impl CoreLoop { partition.set_node_surrogate(src_id, src_surrogate); partition.set_node_surrogate(dst_id, dst_surrogate); self.checkpoint_coordinator.mark_dirty("sparse", 1); - self.note_edge_write_lsn(task, tid, collection, src_id, label, dst_id); - // CDC: emit after `note_edge_write_lsn` so the core + self.note_edge_write(task, tid, collection, src_id, label, dst_id); + // CDC: emit after `note_edge_write` so the core // watermark (the event's LSN) already reflects this // edge's WAL LSN, matching the WAL-replay reconstruction. self.emit_graph_edge_event( diff --git a/nodedb/src/data/executor/handlers/graph_edge_write/put_batch.rs b/nodedb/src/data/executor/handlers/graph_edge_write/put_batch.rs index 3e1fae0f4..157fb5f7f 100644 --- a/nodedb/src/data/executor/handlers/graph_edge_write/put_batch.rs +++ b/nodedb/src/data/executor/handlers/graph_edge_write/put_batch.rs @@ -142,7 +142,7 @@ impl CoreLoop { .mark_dirty("sparse", edges.len()); } for edge in edges { - self.note_edge_write_lsn( + self.note_edge_write( task, tid, edge.collection.as_str(), diff --git a/nodedb/src/data/executor/handlers/graph_edge_write/shared.rs b/nodedb/src/data/executor/handlers/graph_edge_write/shared.rs index de22a0ca1..34a8e7b4e 100644 --- a/nodedb/src/data/executor/handlers/graph_edge_write/shared.rs +++ b/nodedb/src/data/executor/handlers/graph_edge_write/shared.rs @@ -114,7 +114,7 @@ impl CoreLoop { /// Record a committed edge write's version, keyed by the edge's /// `(src, label, dst)` identity, if a WAL LSN was threaded onto the task. - pub(in crate::data::executor) fn note_edge_write_lsn( + pub(in crate::data::executor) fn note_edge_write( &mut self, task: &ExecutionTask, tid: u64, @@ -123,10 +123,10 @@ impl CoreLoop { label: &str, dst_id: &str, ) { - let Some(lsn) = task.wal_lsn() else { + let Some(stamp) = self.task_write_stamp(task) else { return; }; - self.note_write_lsn( + self.note_write( task.request.database_id, TenantId::new(tid), collection, @@ -137,7 +137,7 @@ impl CoreLoop { dst: Box::from(dst_id), }, ), - lsn, + stamp, ); } } @@ -210,6 +210,7 @@ pub(super) mod test_support { wal_lsn: Some(Lsn::new(lsn)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/handlers/graph_node_guard.rs b/nodedb/src/data/executor/handlers/graph_node_guard.rs index d942ce35b..d10cdbaa7 100644 --- a/nodedb/src/data/executor/handlers/graph_node_guard.rs +++ b/nodedb/src/data/executor/handlers/graph_node_guard.rs @@ -293,6 +293,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/handlers/home_versions.rs b/nodedb/src/data/executor/handlers/home_versions.rs index ee2cb32d9..ec48fea78 100644 --- a/nodedb/src/data/executor/handlers/home_versions.rs +++ b/nodedb/src/data/executor/handlers/home_versions.rs @@ -3,17 +3,17 @@ //! `MetaOp::HomeVersions`: this core's current version of each probed home. //! //! The Control Plane hands a core only the probes whose vShard the core owns. -//! A probe with a collection answers the collection's write floor on this -//! core. A probe without one answers the core watermark. Nothing is read -//! beyond those two values, and nothing is written. +//! A probe with a collection answers the collection's version on the probe's +//! vShard. A probe without one answers the vShard's latest version. Nothing +//! is read beyond those two values, and nothing is written. use nodedb_physical::physical_plan::{HomeAnswer, HomeVersion, HomeVersionProbe}; +use nodedb_types::WriteVersion; use crate::bridge::envelope::{ErrorCode, Response}; use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::core_loop::write_index::CollKey; use crate::data::executor::task::ExecutionTask; -use crate::types::Lsn; +use crate::types::VShardId; impl CoreLoop { pub(in crate::data::executor) fn execute_home_versions( @@ -21,13 +21,24 @@ impl CoreLoop { task: &ExecutionTask, probes: &[HomeVersionProbe], ) -> Response { - let answers: Vec = probes - .iter() - .map(|probe| HomeVersion { + let mut answers: Vec = Vec::with_capacity(probes.len()); + for probe in probes { + let Some(version) = self.home_version(task, probe) else { + return self.response_error( + task, + ErrorCode::Internal { + detail: format!( + "home versions: probe names vShard {}, past the vShard space", + probe.vshard + ), + }, + ); + }; + answers.push(HomeVersion { probe: probe.clone(), - answer: HomeAnswer::Version(self.home_version(task, probe).as_u64()), - }) - .collect(); + answer: HomeAnswer::Version(version), + }); + } match zerompk::to_msgpack_vec(&answers) { Ok(payload) => self.response_with_payload(task, payload), Err(e) => self.response_error( @@ -39,17 +50,21 @@ impl CoreLoop { } } - fn home_version(&self, task: &ExecutionTask, probe: &HomeVersionProbe) -> Lsn { - match probe.collection.as_deref() { - Some(collection) => self - .write_index - .collection_write_lsn(&CollKey { - db: task.request.database_id, - tenant: task.request.tenant_id, - collection: Box::from(collection), - }) - .unwrap_or(Lsn::ZERO), - None => self.watermark, + /// The probe's current version, `None` for a vShard id past the vShard + /// space. + fn home_version(&self, task: &ExecutionTask, probe: &HomeVersionProbe) -> Option { + if probe.vshard >= VShardId::COUNT { + return None; } + let vshard = VShardId::new(probe.vshard); + Some(match probe.collection.as_deref() { + Some(collection) => self.write_index.collection_current( + task.request.database_id, + task.request.tenant_id, + vshard, + collection, + ), + None => self.write_index.latest(vshard), + }) } } diff --git a/nodedb/src/data/executor/handlers/join/params.rs b/nodedb/src/data/executor/handlers/join/params.rs index a3d08eaa5..7c9b7d7cb 100644 --- a/nodedb/src/data/executor/handlers/join/params.rs +++ b/nodedb/src/data/executor/handlers/join/params.rs @@ -137,6 +137,7 @@ fn make_dummy_task() -> ExecutionTask { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/handlers/join/shuffle_join.rs b/nodedb/src/data/executor/handlers/join/shuffle_join.rs index 826b35b83..239a58ce8 100644 --- a/nodedb/src/data/executor/handlers/join/shuffle_join.rs +++ b/nodedb/src/data/executor/handlers/join/shuffle_join.rs @@ -216,6 +216,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/handlers/kv/atomic.rs b/nodedb/src/data/executor/handlers/kv/atomic.rs index 40c32d3fc..5bf3d5884 100644 --- a/nodedb/src/data/executor/handlers/kv/atomic.rs +++ b/nodedb/src/data/executor/handlers/kv/atomic.rs @@ -127,7 +127,7 @@ impl CoreLoop { Some(written.as_slice()), None, ); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); match response_codec::encode_json_as_msgpack(&serde_json::json!({ "value": value })) { Ok(payload) => self.response_with_payload(task, payload), @@ -195,7 +195,7 @@ impl CoreLoop { Some(written.as_slice()), None, ); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); match response_codec::encode_json_as_msgpack(&incr_float_reply(value, &written)) { Ok(payload) => self.response_with_payload(task, payload), Err(e) => self.response_error(task, ErrorCode::from(e)), @@ -266,7 +266,7 @@ impl CoreLoop { Some(written.as_slice()), None, ); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); } let current_b64 = result @@ -345,7 +345,7 @@ impl CoreLoop { Some(written.as_slice()), old.as_deref(), ); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); // A row the read policy excludes is reported exactly as an absent row, // the same convention `execute_kv_get` uses — the caller cannot tell it diff --git a/nodedb/src/data/executor/handlers/kv/batch.rs b/nodedb/src/data/executor/handlers/kv/batch.rs index 18fd6fe2c..92394b662 100644 --- a/nodedb/src/data/executor/handlers/kv/batch.rs +++ b/nodedb/src/data/executor/handlers/kv/batch.rs @@ -143,7 +143,7 @@ impl CoreLoop { // version against that single LSN. if task.wal_lsn().is_some() { for (key, _) in entries { - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); } } if let Some(spec) = returning { diff --git a/nodedb/src/data/executor/handlers/kv/crud/write_basic.rs b/nodedb/src/data/executor/handlers/kv/crud/write_basic.rs index 760e50a7e..36bc60aa0 100644 --- a/nodedb/src/data/executor/handlers/kv/crud/write_basic.rs +++ b/nodedb/src/data/executor/handlers/kv/crud/write_basic.rs @@ -68,7 +68,7 @@ impl CoreLoop { }; self.emit_kv_write_event(task, collection, op, key, Some(value), old_slice); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); if let Some(spec) = returning { // `value` IS the stored body on this path: the put writes the // caller's bytes verbatim, so projecting them is projecting the @@ -159,7 +159,7 @@ impl CoreLoop { None, ); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); if let Some(spec) = returning { return self.kv_stored_returning_response(task, spec, rls_filters, &[(key, value)]); } @@ -241,7 +241,7 @@ impl CoreLoop { None, ); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); if let Some(spec) = returning { return self.kv_stored_returning_response(task, spec, rls_filters, &[(key, value)]); } @@ -251,7 +251,7 @@ impl CoreLoop { /// Record a committed KV point write's version, keyed by the raw KV key /// bytes, if a WAL LSN was threaded onto the task. Shared by every KV write /// chokepoint (basic put/delete, atomic ops, batch, field, TTL). - pub(in crate::data::executor) fn note_kv_write_lsn( + pub(in crate::data::executor) fn note_kv_write( &mut self, task: &ExecutionTask, did: u64, @@ -259,15 +259,15 @@ impl CoreLoop { collection: &str, key: &[u8], ) { - let Some(lsn) = task.wal_lsn() else { + let Some(stamp) = self.task_write_stamp(task) else { return; }; - self.note_write_lsn( + self.note_write( crate::types::DatabaseId::new(did), crate::types::TenantId::new(tid), collection, Some(crate::data::executor::core_loop::write_index::KeyRepr::KvKey(Box::from(key))), - lsn, + stamp, ); } } diff --git a/nodedb/src/data/executor/handlers/kv/field.rs b/nodedb/src/data/executor/handlers/kv/field.rs index d5a68451e..4c3f57292 100644 --- a/nodedb/src/data/executor/handlers/kv/field.rs +++ b/nodedb/src/data/executor/handlers/kv/field.rs @@ -201,7 +201,7 @@ impl CoreLoop { Some(computed.new_value.as_slice()), current.as_deref(), ); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); if let Some(spec) = returning { // `computed.new_value` IS the stored body: the merge is persisted // verbatim, so projecting it is projecting the post-image. diff --git a/nodedb/src/data/executor/handlers/kv/predicate/apply.rs b/nodedb/src/data/executor/handlers/kv/predicate/apply.rs index 7ca5020ec..a623d6718 100644 --- a/nodedb/src/data/executor/handlers/kv/predicate/apply.rs +++ b/nodedb/src/data/executor/handlers/kv/predicate/apply.rs @@ -105,7 +105,7 @@ impl CoreLoop { Some(new_value), Some(old_body), ); - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); } if let Some(spec) = returning { diff --git a/nodedb/src/data/executor/handlers/kv/resolve/apply.rs b/nodedb/src/data/executor/handlers/kv/resolve/apply.rs index 1e345a348..182673f8c 100644 --- a/nodedb/src/data/executor/handlers/kv/resolve/apply.rs +++ b/nodedb/src/data/executor/handlers/kv/resolve/apply.rs @@ -148,7 +148,7 @@ impl CoreLoop { Some(value), precondition.as_deref(), ); - self.note_kv_write_lsn(task, did, tid, collection.as_str(), key); + self.note_kv_write(task, did, tid, collection.as_str(), key); } // The drift check confirmed the row holds `precondition`, so it is // present and keeps its bound identity. @@ -183,7 +183,7 @@ impl CoreLoop { Some(value), Some(precondition.as_slice()), ); - self.note_kv_write_lsn(task, did, tid, collection.as_str(), key); + self.note_kv_write(task, did, tid, collection.as_str(), key); } KvResolvedMutation::Delete { collection, @@ -203,7 +203,7 @@ impl CoreLoop { None, precondition.as_deref(), ); - self.note_kv_write_lsn(task, did, tid, collection.as_str(), key); + self.note_kv_write(task, did, tid, collection.as_str(), key); } // No body change, so no event — matches the live handlers. KvResolvedMutation::Expire { @@ -220,7 +220,7 @@ impl CoreLoop { key, resolved_now_ms.saturating_add(*ttl_ms), ); - self.note_kv_write_lsn(task, did, tid, collection.as_str(), key); + self.note_kv_write(task, did, tid, collection.as_str(), key); } KvResolvedMutation::Persist { collection, @@ -228,7 +228,7 @@ impl CoreLoop { precondition: _, } => { self.kv_engine.persist(did, tid, collection.as_str(), key); - self.note_kv_write_lsn(task, did, tid, collection.as_str(), key); + self.note_kv_write(task, did, tid, collection.as_str(), key); } } Ok(()) diff --git a/nodedb/src/data/executor/handlers/kv/ttl.rs b/nodedb/src/data/executor/handlers/kv/ttl.rs index 7d71934ba..4d9e1bf8b 100644 --- a/nodedb/src/data/executor/handlers/kv/ttl.rs +++ b/nodedb/src/data/executor/handlers/kv/ttl.rs @@ -69,7 +69,7 @@ impl CoreLoop { .kv_engine .expire(did, tid, collection, key, ttl_ms, now_ms) { - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); self.response_ok(task) } else { self.response_error(task, ErrorCode::NotFound) @@ -99,7 +99,7 @@ impl CoreLoop { } if self.kv_engine.persist(did, tid, collection, key) { - self.note_kv_write_lsn(task, did, tid, collection, key); + self.note_kv_write(task, did, tid, collection, key); self.response_ok(task) } else { self.response_error(task, ErrorCode::NotFound) diff --git a/nodedb/src/data/executor/handlers/point/apply_put/core.rs b/nodedb/src/data/executor/handlers/point/apply_put/core.rs index 8802fdb63..097803849 100644 --- a/nodedb/src/data/executor/handlers/point/apply_put/core.rs +++ b/nodedb/src/data/executor/handlers/point/apply_put/core.rs @@ -469,6 +469,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, }) } diff --git a/nodedb/src/data/executor/handlers/point/apply_put/index.rs b/nodedb/src/data/executor/handlers/point/apply_put/index.rs index 0521c89c7..cecd41aa3 100644 --- a/nodedb/src/data/executor/handlers/point/apply_put/index.rs +++ b/nodedb/src/data/executor/handlers/point/apply_put/index.rs @@ -267,6 +267,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, }) } diff --git a/nodedb/src/data/executor/handlers/point/delete.rs b/nodedb/src/data/executor/handlers/point/delete.rs index 9dbb4a651..1cc5cfc7c 100644 --- a/nodedb/src/data/executor/handlers/point/delete.rs +++ b/nodedb/src/data/executor/handlers/point/delete.rs @@ -181,11 +181,11 @@ impl CoreLoop { // collection, but only when a row was actually removed — a delete that // matched nothing changes no state and creates no OCC conflict. if prior.is_some() { - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); // Record the removed secondary-index values into the per-index // write-value substrate (plain cascade ∪ bitemporal tombstones). - if let Some(lsn) = task.wal_lsn() { + if let Some(stamp) = self.task_write_stamp(task) { let mut tuples = outcome.secondary_index_tuples; tuples.extend(outcome.bitemporal_index_tuples); self.note_index_write_values( @@ -193,7 +193,7 @@ impl CoreLoop { crate::types::TenantId::new(tid), collection, &tuples, - lsn, + stamp, ); } } diff --git a/nodedb/src/data/executor/handlers/point/insert.rs b/nodedb/src/data/executor/handlers/point/insert.rs index 8ad6005e8..c1145eb45 100644 --- a/nodedb/src/data/executor/handlers/point/insert.rs +++ b/nodedb/src/data/executor/handlers/point/insert.rs @@ -298,7 +298,7 @@ impl CoreLoop { self.checkpoint_coordinator.mark_dirty("sparse", 1); - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); // The exact bytes storage now holds, taken before the index tuples are // consumed below. A `RETURNING` projection reads these rather than @@ -308,7 +308,7 @@ impl CoreLoop { // Record the touched secondary-index values into the per-index // write-value substrate (added ∪ removed ∪ bitemporal tuples). - if let Some(lsn) = task.wal_lsn() { + if let Some(stamp) = self.task_write_stamp(task) { let mut tuples = outcome.secondary_index_added; tuples.extend(outcome.secondary_index_removed); tuples.extend(outcome.bitemporal_index_tuples); @@ -317,7 +317,7 @@ impl CoreLoop { crate::types::TenantId::new(tid), collection, &tuples, - lsn, + stamp, ); } diff --git a/nodedb/src/data/executor/handlers/point/put.rs b/nodedb/src/data/executor/handlers/point/put.rs index ff7b64753..255d0337d 100644 --- a/nodedb/src/data/executor/handlers/point/put.rs +++ b/nodedb/src/data/executor/handlers/point/put.rs @@ -200,11 +200,11 @@ impl CoreLoop { } // Record the committed write's version against its surrogate + collection. - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); // Record the touched secondary-index values into the per-index // write-value substrate (added ∪ removed ∪ bitemporal tuples). - if let Some(lsn) = task.wal_lsn() { + if let Some(stamp) = self.task_write_stamp(task) { let mut tuples = std::mem::take(&mut prior.secondary_index_added); tuples.append(&mut prior.secondary_index_removed); tuples.append(&mut prior.bitemporal_index_tuples); @@ -213,7 +213,7 @@ impl CoreLoop { crate::types::TenantId::new(tid), collection, &tuples, - lsn, + stamp, ); } diff --git a/nodedb/src/data/executor/handlers/point/update/exec.rs b/nodedb/src/data/executor/handlers/point/update/exec.rs index 5e396b166..818a2f040 100644 --- a/nodedb/src/data/executor/handlers/point/update/exec.rs +++ b/nodedb/src/data/executor/handlers/point/update/exec.rs @@ -242,10 +242,15 @@ impl CoreLoop { bitemporal, sys_from_ms: sys_from_for_encode, wal_lsn: task.wal_lsn(), + write_stamp: self.task_write_stamp(task), resolved_sum_targets, }); match write_result { Ok(target_write_set) => { + // Record the committed row's write version against + // its surrogate and collection. A read of the row + // from before this update then fails validation. + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); self.doc_cache.put( task.request.database_id.as_u64(), tid, diff --git a/nodedb/src/data/executor/handlers/point/update/persist.rs b/nodedb/src/data/executor/handlers/point/update/persist.rs index 8ed06e1eb..a4021563d 100644 --- a/nodedb/src/data/executor/handlers/point/update/persist.rs +++ b/nodedb/src/data/executor/handlers/point/update/persist.rs @@ -40,6 +40,10 @@ pub(in crate::data::executor) struct PointUpdatePersist<'a> { pub(in crate::data::executor) bitemporal: bool, pub(in crate::data::executor) sys_from_ms: i64, pub(in crate::data::executor) wal_lsn: Option, + /// The stamp the update's index values record under, `None` without a + /// WAL record. + pub(in crate::data::executor) write_stamp: + Option, /// `(target collection, join-key value)` → target row surrogate for every /// materialized-sum target this update may touch — BOTH sides when the /// update moves a row between targets by changing its join key. Resolved on @@ -70,6 +74,7 @@ impl CoreLoop { bitemporal, sys_from_ms, wal_lsn, + write_stamp, resolved_sum_targets, } = params; @@ -169,7 +174,7 @@ impl CoreLoop { // Index write-versions are published only once the write they describe // is durable. - if let Some(lsn) = wal_lsn + if let Some(stamp) = write_stamp && !touched.is_empty() { self.note_index_write_values( @@ -177,7 +182,7 @@ impl CoreLoop { TenantId::new(tid), collection, &touched, - lsn, + stamp, ); } diff --git a/nodedb/src/data/executor/handlers/query_collection_size.rs b/nodedb/src/data/executor/handlers/query_collection_size.rs index 6916001ba..ffb2fb116 100644 --- a/nodedb/src/data/executor/handlers/query_collection_size.rs +++ b/nodedb/src/data/executor/handlers/query_collection_size.rs @@ -80,7 +80,7 @@ impl CoreLoop { watermark_lsn: self.watermark, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/data/executor/handlers/snapshot/capture_arrays.rs b/nodedb/src/data/executor/handlers/snapshot/capture_arrays.rs index 74cd92e39..ab14cad02 100644 --- a/nodedb/src/data/executor/handlers/snapshot/capture_arrays.rs +++ b/nodedb/src/data/executor/handlers/snapshot/capture_arrays.rs @@ -3,15 +3,15 @@ //! Tenant snapshot capture of the array engine. //! //! Every cell version of each of the tenant's arrays in the snapshot's -//! database is exported, grouped by the vShard its Hilbert prefix routes to. +//! database is exported, grouped by the vShard its tile's key routes to. //! A backup keeps each group on the source node of that vShard, so every //! version is captured once. use std::collections::BTreeMap; use std::sync::Arc; -use nodedb_array::coord::encode::encode_hilbert_prefix; use nodedb_array::schema::ArraySchema; +use nodedb_array::tile::cell_tile_prefix; use nodedb_cluster::distributed_array::array_vshard_for_tile; use crate::data::executor::core_loop::CoreLoop; @@ -81,7 +81,7 @@ fn by_vshard( ) -> crate::Result>> { let mut grouped: BTreeMap> = BTreeMap::new(); for version in versions { - let prefix = match encode_hilbert_prefix(schema, &version.coord) { + let prefix = match cell_tile_prefix(schema, &version.coord) { Ok(prefix) => prefix, Err(_) if version.payload.is_none() => continue, Err(e) => { @@ -152,7 +152,7 @@ mod tests { ); for (vshard, cells) in &grouped { for cell in cells { - let prefix = encode_hilbert_prefix(&s, &cell.coord).expect("prefix"); + let prefix = cell_tile_prefix(&s, &cell.coord).expect("prefix"); assert_eq!(array_vshard_for_tile(prefix, 8).expect("route"), *vshard); } } diff --git a/nodedb/src/data/executor/handlers/snapshot/install_arrays.rs b/nodedb/src/data/executor/handlers/snapshot/install_arrays.rs index 42421eaf6..63b16c4e0 100644 --- a/nodedb/src/data/executor/handlers/snapshot/install_arrays.rs +++ b/nodedb/src/data/executor/handlers/snapshot/install_arrays.rs @@ -10,8 +10,8 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; -use nodedb_array::coord::encode::encode_hilbert_prefix; use nodedb_array::schema::ArraySchema; +use nodedb_array::tile::cell_tile_prefix; use nodedb_array::types::coord::value::CoordValue; use nodedb_cluster::distributed_array::array_vshard_for_tile; @@ -93,7 +93,7 @@ impl CoreLoop { let routes = Arc::clone(&schema); let prefix_bits = entry.prefix_bits; let replaced = |coord: &[CoordValue]| { - encode_hilbert_prefix(&routes, coord) + cell_tile_prefix(&routes, coord) .ok() .and_then(|prefix| array_vshard_for_tile(prefix, prefix_bits).ok()) .is_some_and(|vshard| group.contains(&vshard)) diff --git a/nodedb/src/data/executor/handlers/spatial/rtree_scan.rs b/nodedb/src/data/executor/handlers/spatial/rtree_scan.rs index 5c77fd621..9acf744fa 100644 --- a/nodedb/src/data/executor/handlers/spatial/rtree_scan.rs +++ b/nodedb/src/data/executor/handlers/spatial/rtree_scan.rs @@ -388,6 +388,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/handlers/spatial_sync.rs b/nodedb/src/data/executor/handlers/spatial_sync.rs index c6d05df41..fcbe5e9e0 100644 --- a/nodedb/src/data/executor/handlers/spatial_sync.rs +++ b/nodedb/src/data/executor/handlers/spatial_sync.rs @@ -199,7 +199,7 @@ impl CoreLoop { self.spatial_doc_map.insert(doc_map_key, doc_id); // Advance the collection floor for this committed spatial write. - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); // Advance HWM and return gate result payload when provenance is present. if let Some(prov) = provenance { @@ -297,7 +297,7 @@ impl CoreLoop { self.spatial_doc_map.remove(&doc_map_key); // Advance the collection floor for this committed spatial delete. - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); if let Some(prov) = provenance { self.sync_commit(prov); diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest.rs b/nodedb/src/data/executor/handlers/timeseries/ingest.rs index 875da439b..e4d96e440 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest.rs @@ -432,7 +432,7 @@ impl CoreLoop { watermark_lsn: self.watermark, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest_dispatch.rs b/nodedb/src/data/executor/handlers/timeseries/ingest_dispatch.rs index f10ac9ab3..63c5585f0 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest_dispatch.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest_dispatch.rs @@ -174,7 +174,7 @@ impl CoreLoop { watermark_lsn: self.watermark, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }; } @@ -265,7 +265,7 @@ impl CoreLoop { return self.sync_ack_response(task, AckStatus::Applied, applied_seq); } if ingest_response.status == Status::Ok { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } ingest_response } diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest_resolved.rs b/nodedb/src/data/executor/handlers/timeseries/ingest_resolved.rs index 7839ce88b..e516dc301 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest_resolved.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest_resolved.rs @@ -383,7 +383,7 @@ impl CoreLoop { watermark_lsn: self.watermark, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }, Err(e) => self.response_error(task, ErrorCode::from(e)), diff --git a/nodedb/src/data/executor/handlers/timeseries/scan.rs b/nodedb/src/data/executor/handlers/timeseries/scan.rs index 35475a182..399723f5b 100644 --- a/nodedb/src/data/executor/handlers/timeseries/scan.rs +++ b/nodedb/src/data/executor/handlers/timeseries/scan.rs @@ -275,7 +275,7 @@ impl CoreLoop { watermark_lsn: self.watermark, error_code: None, stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), } } diff --git a/nodedb/src/data/executor/handlers/timeseries/truncate.rs b/nodedb/src/data/executor/handlers/timeseries/truncate.rs index b1a7f8fbd..05aa0eb8f 100644 --- a/nodedb/src/data/executor/handlers/timeseries/truncate.rs +++ b/nodedb/src/data/executor/handlers/timeseries/truncate.rs @@ -165,7 +165,7 @@ impl CoreLoop { self.checkpoint_coordinator .mark_dirty("timeseries", truncated as usize); - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); self.invalidate_aggregate_cache_for_collection(db.as_u64(), tid.as_u64(), collection); debug!(core = self.core_id, %collection, truncated, "timeseries truncate complete"); diff --git a/nodedb/src/data/executor/handlers/timeseries_wal.rs b/nodedb/src/data/executor/handlers/timeseries_wal.rs index bb888678a..6e5ee75ca 100644 --- a/nodedb/src/data/executor/handlers/timeseries_wal.rs +++ b/nodedb/src/data/executor/handlers/timeseries_wal.rs @@ -306,7 +306,23 @@ impl CoreLoop { mod tests { use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::write_index::CollKey; - use crate::types::{DatabaseId, Lsn, TenantId}; + use crate::data::executor::core_loop::write_index::tests::local; + use crate::types::{DatabaseId, TenantId}; + + /// The collection key a replayed write of `collection` in database 0 + /// records under: the collection's home vShard. + fn replayed_key(collection: &str) -> CollKey { + let vshard = + nodedb_types::CollectionKey::from_qualified_str(DatabaseId::new(0), collection) + .expect("collection key") + .vshard(); + CollKey { + vshard, + db: DatabaseId::new(0), + tenant: TenantId::new(7), + collection: Box::from(collection), + } + } use crate::wal::{DecodedBatchRecord, decode_batch_record}; use nodedb_types::Surrogate; use nodedb_types::columnar::ColumnarWalRecord; @@ -557,14 +573,14 @@ mod tests { } /// WAL replay threads the record LSN into `replay_task` so - /// `execute_columnar_insert`'s `note_collection_write_lsn(task, ..)` call + /// `execute_columnar_insert`'s `note_collection_write(task, ..)` call /// (gated on `task.wal_lsn().is_some()`) fires during WAL replay too, not - /// just on live writes. This proves the collection floor in + /// just on live writes. This proves the collection version in /// `WriteVersionIndex` is populated end-to-end through /// `replay_timeseries_wal` -> `replay_columnar_payload` -> /// `execute_columnar_insert`. #[test] - fn columnar_insert_replay_populates_collection_write_lsn_floor() { + fn columnar_insert_replay_populates_the_collection_write_version() { let mut h = make_core(); let record = columnar_wal_record("events_wv", 123, 7); @@ -574,16 +590,13 @@ mod tests { &nodedb_wal::TombstoneSet::new(), ); - let coll_key = CollKey { - db: DatabaseId::new(0), - tenant: TenantId::new(7), - collection: Box::from("events_wv"), - }; assert_eq!( - h.core.write_index.collection_write_lsn(&coll_key), - Some(Lsn::new(123)), - "columnar insert replay must record the record LSN as the \ - collection write-version floor" + h.core + .write_index + .collection_version(&replayed_key("events_wv")), + Some(local(123)), + "columnar insert replay must record the record's version as the \ + collection write version" ); } @@ -630,14 +643,11 @@ mod tests { &nodedb_wal::TombstoneSet::new(), ); - let coll_key = CollKey { - db: DatabaseId::new(0), - tenant: TenantId::new(7), - collection: Box::from("events_ungated"), - }; assert_eq!( - h.core.write_index.collection_write_lsn(&coll_key), - Some(Lsn::new(150)), + h.core + .write_index + .collection_version(&replayed_key("events_ungated")), + Some(local(150)), "a record above the floor must be applied" ); } diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs index 1f464607b..c6ed0201f 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/calvin_install_tests.rs @@ -267,15 +267,20 @@ fn a_calvin_install_notes_index_values_at_the_redo_lsn() { assert_eq!(installed.status, Status::Ok, "{:?}", installed.error_code); assert_eq!( - core.write_index.index_values.value_lsn( - DatabaseId::DEFAULT, - TenantId::new(TID), - ORDERS, - "a", + core.write_index.index_values.value_version( + crate::data::executor::core_loop::index_value_versions::IndexDimRef { + vshard: crate::types::VShardId::new(0), + db: DatabaseId::DEFAULT, + tenant: TenantId::new(TID), + collection: ORDERS, + field: "a", + }, "1" ), - Some(Lsn::new(43)), - "the install records the row's index value at the redo LSN" + Some(crate::data::executor::core_loop::write_index::tests::local( + 43 + )), + "the install records the row's index value at the redo's version" ); } diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/entry.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/entry.rs index 4a0ac94de..edd5d42f2 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/entry.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/entry.rs @@ -163,7 +163,12 @@ impl CoreLoop { if let Err(refusal) = self.validate_redo_pass(&target, committed.sum_targets) { return self.response_error(task, refusal.into_code()); } + // The replay arms install the record. Each write they apply takes the + // record's stamp: its vShard and the entry it applies. + let stamp = self.task_write_stamp_at(task, lsn); + self.write_index.record_stamps.insert(stamp); let installed = self.install_redo_pass(&target, committed.sum_targets); + self.write_index.record_stamps.remove(lsn); let mut scope = match installed { Ok(scope) => scope, Err(refusal) => return self.response_error(task, refusal.into_code()), @@ -198,7 +203,7 @@ impl CoreLoop { version.tenant, &version.collection, version.key, - version.lsn, + version.stamp, ); } // Events leave only once the record is settled. Every @@ -232,7 +237,7 @@ impl CoreLoop { let tenant = TenantId::new(tid); for collection in committed.collections { - self.note_write_lsn(task.request.database_id, tenant, collection, None, lsn); + self.note_write(task.request.database_id, tenant, collection, None, stamp); } for write in &scope.doc_writes { self.note_index_write_values( @@ -240,7 +245,7 @@ impl CoreLoop { tenant, &write.collection, &write.index_tuples, - lsn, + stamp, ); } let write_set = target_write_set(&scope.target_writes); diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/install_refusal_tests.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/install_refusal_tests.rs index cacb3654d..4559bffd4 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/install_refusal_tests.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/install_refusal_tests.rs @@ -521,12 +521,17 @@ fn a_refused_install_publishes_no_write_version_and_no_watermark() { let point = nodedb_types::calvin::ReadKeyIdent::Point(crate::types::KeyRepr::Surrogate(41)); let committed = run(&[document_insert()], false, |_| {}); assert_eq!(committed.core.watermark, Lsn::new(200)); + // The install runs on the default task's vShard 0 and applies no + // data-group entry, so its version is its WAL LSN there. + let vshard = crate::types::VShardId::new(0); + let before_install = crate::data::executor::core_loop::write_index::tests::local(199); assert!(!committed.core.write_index.read_is_valid( DatabaseId::DEFAULT, TenantId::new(TID), + vshard, "notes", &point, - Lsn::new(199), + before_install, )); let refused = run(&[document_insert()], true, |_| {}); @@ -538,9 +543,10 @@ fn a_refused_install_publishes_no_write_version_and_no_watermark() { refused.core.write_index.read_is_valid( DatabaseId::DEFAULT, TenantId::new(TID), + vshard, "notes", &point, - Lsn::new(199), + before_install, ), "the rolled-back install publishes no write version" ); diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs index df380a241..6de4a417d 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/state.rs @@ -19,11 +19,10 @@ use nodedb_physical::physical_plan::{RedoSumTargets, ResolvedSumTarget}; use nodedb_types::RowIdentity; use crate::bridge::envelope::ErrorCode; -use crate::data::executor::core_loop::write_index::KeyRepr; +use crate::data::executor::core_loop::write_index::{KeyRepr, WriteStamp}; use crate::data::executor::enforcement::materialized_sum::apply::TargetWrite; use crate::data::executor::handlers::transaction::undo::UndoEntry; use crate::event::WriteOp; -use crate::types::Lsn; /// Committed-redo apply state owned by one core. pub(in crate::data::executor) struct RedoApplyState { @@ -123,7 +122,7 @@ pub(in crate::data::executor) struct DeferredWriteVersion { pub tenant: crate::types::TenantId, pub collection: String, pub key: Option, - pub lsn: Lsn, + pub stamp: WriteStamp, } /// `(database, tenant, collection)`. @@ -173,7 +172,7 @@ impl RedoApplyScope { tenant: crate::types::TenantId, collection: &str, key: Option, - lsn: Lsn, + stamp: WriteStamp, ) { if self.pass == RedoApplyPass::Install { self.write_versions.push(DeferredWriteVersion { @@ -181,7 +180,7 @@ impl RedoApplyScope { tenant, collection: collection.to_string(), key, - lsn, + stamp, }); } } diff --git a/nodedb/src/data/executor/handlers/transaction/resolve/entry.rs b/nodedb/src/data/executor/handlers/transaction/resolve/entry.rs index 241f5a608..aea64aad8 100644 --- a/nodedb/src/data/executor/handlers/transaction/resolve/entry.rs +++ b/nodedb/src/data/executor/handlers/transaction/resolve/entry.rs @@ -548,6 +548,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_graph.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_graph.rs index 7fe8e9778..b6c39cebe 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_graph.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_graph.rs @@ -447,6 +447,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::AlreadyOrdered), }, state: TaskState::Running, diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv.rs index 0a880f5bc..c707132d3 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv.rs @@ -426,6 +426,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }; ExecutionTask::new(request) diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs index c8eb39688..c8c0ffa58 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs @@ -295,6 +295,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_timeseries.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_timeseries.rs index 6f1cac54a..2377c1f6d 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_timeseries.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_timeseries.rs @@ -553,6 +553,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::AlreadyOrdered), }, state: TaskState::Running, diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_vector.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_vector.rs index 808645e84..3777b1588 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_vector.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_vector.rs @@ -515,6 +515,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/handlers/transaction/undo/apply.rs b/nodedb/src/data/executor/handlers/transaction/undo/apply.rs index 45e145c2b..d6dac454a 100644 --- a/nodedb/src/data/executor/handlers/transaction/undo/apply.rs +++ b/nodedb/src/data/executor/handlers/transaction/undo/apply.rs @@ -602,6 +602,7 @@ mod tests { wal_lsn: Some(Lsn::new(lsn)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::AlreadyOrdered), }, Some(Lsn::new(lsn)), diff --git a/nodedb/src/data/executor/handlers/transaction/write_version.rs b/nodedb/src/data/executor/handlers/transaction/write_version.rs index 2d76e597b..b991ce8b1 100644 --- a/nodedb/src/data/executor/handlers/transaction/write_version.rs +++ b/nodedb/src/data/executor/handlers/transaction/write_version.rs @@ -3,13 +3,14 @@ //! The per-key write versions a committed batch of plans implies, read off //! the plans. The redo install records versions through its replay arms. //! The parity tests check the install against this oracle, engine by engine. -//! One WAL LSN applies to every key in the batch. +//! One write stamp applies to every key in the batch. use crate::bridge::envelope::PhysicalPlan; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::write_index::KeyRepr; +use crate::data::executor::core_loop::write_index::WriteStamp; use crate::data::executor::task::ExecutionTask; -use crate::types::{Lsn, TenantId}; +use crate::types::TenantId; use nodedb_physical::physical_plan::{ ColumnarOp, CrdtOp, DocumentOp, GraphOp, SpatialOp, TextOp, TimeseriesOp, VectorOp, VectorWriteTargets, @@ -25,13 +26,13 @@ impl CoreLoop { tid: u64, plans: &[PhysicalPlan], ) { - let Some(lsn) = task.wal_lsn() else { + let Some(stamp) = self.task_write_stamp(task) else { return; }; let db = task.request.database_id; let tenant = TenantId::new(tid); for plan in plans { - self.record_plan_write_version(db, tenant, plan, lsn); + self.record_plan_write_version(db, tenant, plan, stamp); } } @@ -40,13 +41,13 @@ impl CoreLoop { db: crate::types::DatabaseId, tenant: TenantId, plan: &PhysicalPlan, - lsn: Lsn, + stamp: WriteStamp, ) { match plan { - PhysicalPlan::Document(op) => self.record_document_version(db, tenant, op, lsn), - PhysicalPlan::Vector(op) => self.record_vector_version(db, tenant, op, lsn), - PhysicalPlan::Graph(op) => self.record_graph_version(db, tenant, op, lsn), - PhysicalPlan::Kv(op) => self.record_kv_version(db, tenant, op, lsn), + PhysicalPlan::Document(op) => self.record_document_version(db, tenant, op, stamp), + PhysicalPlan::Vector(op) => self.record_vector_version(db, tenant, op, stamp), + PhysicalPlan::Graph(op) => self.record_graph_version(db, tenant, op, stamp), + PhysicalPlan::Kv(op) => self.record_kv_version(db, tenant, op, stamp), // Collection-floor engines: per-key identity is engine-internal. PhysicalPlan::Columnar(op) => { let coll = match op { @@ -63,13 +64,13 @@ impl CoreLoop { ColumnarOp::ResolveDml { .. } => None, }; if let Some(c) = coll { - self.note_write_lsn(db, tenant, c, None, lsn); + self.note_write(db, tenant, c, None, stamp); } } PhysicalPlan::Timeseries( TimeseriesOp::Ingest { collection, .. } | TimeseriesOp::Truncate { collection, .. }, ) => { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); } PhysicalPlan::Spatial(op) => { let coll = match op { @@ -80,7 +81,7 @@ impl CoreLoop { SpatialOp::Scan { .. } => None, }; if let Some(c) = coll { - self.note_write_lsn(db, tenant, c, None, lsn); + self.note_write(db, tenant, c, None, stamp); } } PhysicalPlan::Text(op) => { @@ -97,13 +98,13 @@ impl CoreLoop { TextOp::SetTextConfig { .. } => None, }; if let Some(c) = coll { - self.note_write_lsn(db, tenant, c, None, lsn); + self.note_write(db, tenant, c, None, stamp); } } PhysicalPlan::Crdt(CrdtOp::Apply { collection, .. }) | PhysicalPlan::Crdt(CrdtOp::DocUpsert { collection, .. }) | PhysicalPlan::Crdt(CrdtOp::DocDelete { collection, .. }) => { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); } // No per-key/collection version recorded for reads, control ops, or // engines not (yet) part of the funnel (array is keyed by tile). @@ -122,7 +123,7 @@ impl CoreLoop { db: crate::types::DatabaseId, tenant: TenantId, op: &DocumentOp, - lsn: Lsn, + stamp: WriteStamp, ) { let (collection, surrogate) = match op { DocumentOp::PointPut { @@ -147,7 +148,7 @@ impl CoreLoop { // Whole-collection mutation: key set is every row, so only the // collection floor applies. DocumentOp::Truncate { collection, .. } => { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); return; } // No per-key version recorded: reads, DDL/index ops, and @@ -174,12 +175,12 @@ impl CoreLoop { | DocumentOp::ResolveWrite(_) | DocumentOp::ResolvedWrite { .. } => return, }; - self.note_write_lsn( + self.note_write( db, tenant, collection, Some(KeyRepr::Surrogate(surrogate.as_u32())), - lsn, + stamp, ); } @@ -193,7 +194,7 @@ impl CoreLoop { db: crate::types::DatabaseId, tenant: TenantId, op: &VectorOp, - lsn: Lsn, + stamp: WriteStamp, ) { match op { VectorOp::Insert { @@ -216,12 +217,12 @@ impl CoreLoop { surrogate, .. } => { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), Some(KeyRepr::Surrogate(surrogate.as_u32())), - lsn, + stamp, ); } // An unbound delete names no row: it records the collection @@ -231,12 +232,12 @@ impl CoreLoop { surrogate, .. } => { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), surrogate.map(|s| KeyRepr::Surrogate(s.as_u32())), - lsn, + stamp, ); } VectorOp::MultiVectorInsert { @@ -249,12 +250,12 @@ impl CoreLoop { document_surrogate, .. } => { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), Some(KeyRepr::Surrogate(document_surrogate.as_u32())), - lsn, + stamp, ); } VectorOp::BatchInsert { @@ -265,21 +266,21 @@ impl CoreLoop { // Every vector of a batch carries its bound surrogate. An // empty batch records the collection floor only. for s in surrogates { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), Some(KeyRepr::Surrogate(s.as_u32())), - lsn, + stamp, ); } if surrogates.is_empty() { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); } } // Every row of the collection: the collection floor only. VectorOp::DirectTruncate { collection, .. } => { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); } // A point-targeted delete or update names its surrogates; a // predicate-targeted one resolves them on apply, so it records the @@ -296,17 +297,17 @@ impl CoreLoop { } => match targets { VectorWriteTargets::Surrogates(surrogates) if !surrogates.is_empty() => { for s in surrogates { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), Some(KeyRepr::Surrogate(s.as_u32())), - lsn, + stamp, ); } } VectorWriteTargets::Surrogates(_) | VectorWriteTargets::Predicate(_) => { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); } }, // Every resolved mutation names its surrogate. @@ -316,16 +317,16 @@ impl CoreLoop { .. } => { for mutation in mutations { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), Some(KeyRepr::Surrogate(mutation.surrogate().as_u32())), - lsn, + stamp, ); } if mutations.is_empty() { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); } } // Sparse (doc_id-keyed) and `Delete` (vector_id isn't the @@ -333,7 +334,7 @@ impl CoreLoop { VectorOp::SparseInsert { collection, .. } | VectorOp::SparseDelete { collection, .. } | VectorOp::Delete { collection, .. } => { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); } // Read / config / query ops: nothing written, no version to record. // The resolve pass reads what a governed write depends on. @@ -356,7 +357,7 @@ impl CoreLoop { db: crate::types::DatabaseId, tenant: TenantId, op: &GraphOp, - lsn: Lsn, + stamp: WriteStamp, ) { match op { GraphOp::EdgePut { @@ -373,7 +374,7 @@ impl CoreLoop { dst_id, .. } => { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), @@ -382,12 +383,12 @@ impl CoreLoop { label: Box::from(label.as_str()), dst: Box::from(dst_id.as_str()), }), - lsn, + stamp, ); } GraphOp::EdgePutBatch { edges } | GraphOp::EdgeDeleteBatch { edges } => { for edge in edges { - self.note_write_lsn( + self.note_write( db, tenant, edge.collection.as_str(), @@ -396,7 +397,7 @@ impl CoreLoop { label: Box::from(edge.label.as_str()), dst: Box::from(edge.dst_id.as_str()), }), - lsn, + stamp, ); } } diff --git a/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs b/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs index fc8cfc11d..d2d9ef4b8 100644 --- a/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs +++ b/nodedb/src/data/executor/handlers/transaction/write_version_kv.rs @@ -1,12 +1,13 @@ // SPDX-License-Identifier: BUSL-1.1 //! `KvOp` write versions for the parity oracle in `write_version.rs`. Same -//! contract: no-op with no WAL LSN, one `note_write_lsn` per key a committed +//! contract: no-op with no WAL LSN, one `note_write` per key a committed //! KV write touched. use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::write_index::KeyRepr; -use crate::types::{Lsn, TenantId}; +use crate::data::executor::core_loop::write_index::WriteStamp; +use crate::types::TenantId; use nodedb_physical::physical_plan::KvOp; impl CoreLoop { @@ -15,7 +16,7 @@ impl CoreLoop { db: crate::types::DatabaseId, tenant: TenantId, op: &KvOp, - lsn: Lsn, + stamp: WriteStamp, ) { match op { KvOp::Put { @@ -51,24 +52,24 @@ impl CoreLoop { | KvOp::GetSet { collection, key, .. } => { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), Some(KeyRepr::KvKey(Box::from(key.as_slice()))), - lsn, + stamp, ); } KvOp::Delete { collection, keys, .. } => { for key in keys { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), Some(KeyRepr::KvKey(Box::from(key.as_slice()))), - lsn, + stamp, ); } } @@ -78,12 +79,12 @@ impl CoreLoop { .. } => { for (key, _value) in entries { - self.note_write_lsn( + self.note_write( db, tenant, collection.as_str(), Some(KeyRepr::KvKey(Box::from(key.as_slice()))), - lsn, + stamp, ); } } @@ -92,7 +93,7 @@ impl CoreLoop { KvOp::Truncate { collection, .. } | KvOp::PredicateUpdate { collection, .. } | KvOp::PredicateDelete { collection, .. } => { - self.note_write_lsn(db, tenant, collection.as_str(), None, lsn); + self.note_write(db, tenant, collection.as_str(), None, stamp); } // Read-only: nothing written, no version to record. KvOp::Get { .. } diff --git a/nodedb/src/data/executor/handlers/transaction/write_version_parity_tests.rs b/nodedb/src/data/executor/handlers/transaction/write_version_parity_tests.rs index 31196cb32..12245ffa1 100644 --- a/nodedb/src/data/executor/handlers/transaction/write_version_parity_tests.rs +++ b/nodedb/src/data/executor/handlers/transaction/write_version_parity_tests.rs @@ -115,17 +115,17 @@ fn assert_parity(installed: &CoreLoop, plans: &[PhysicalPlan]) { .record_batch_write_versions(&task_at(LSN), tid(), plans); let floors: Vec<_> = oracle.core.write_index.recorded_collections().collect(); assert!(!floors.is_empty(), "the oracle records the slice's writes"); - for (key, lsn) in floors { - let held = installed.write_index.collection_write_lsn(key); + for (key, version) in floors { + let held = installed.write_index.collection_version(key); assert!( - held.is_some_and(|held| held >= lsn), + held.is_some_and(|held| held >= version), "the install records the collection floor of {key:?}: {held:?}" ); } - for (key, lsn) in oracle.core.write_index.recorded_keys() { - let held = installed.write_index.key_write_lsn(key); + for (key, version) in oracle.core.write_index.recorded_keys() { + let held = installed.write_index.key_version(key); assert!( - held.is_some_and(|held| held >= lsn), + held.is_some_and(|held| held >= version), "the install records the version of {key:?}: {held:?}" ); } diff --git a/nodedb/src/data/executor/handlers/truncate.rs b/nodedb/src/data/executor/handlers/truncate.rs index 46d0ed35e..84d3afaf8 100644 --- a/nodedb/src/data/executor/handlers/truncate.rs +++ b/nodedb/src/data/executor/handlers/truncate.rs @@ -229,6 +229,9 @@ impl CoreLoop { declared_primary_key, *storage_key, ); + // Record the removal's version against the row's surrogate + // and collection, as a point delete does. + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); // The row's text leaves the inverted index with every other // row's, in one purge once the loop ends. removed.push(surrogate); diff --git a/nodedb/src/data/executor/handlers/update_from_join_write.rs b/nodedb/src/data/executor/handlers/update_from_join_write.rs index d5b7fa7e7..774557320 100644 --- a/nodedb/src/data/executor/handlers/update_from_join_write.rs +++ b/nodedb/src/data/executor/handlers/update_from_join_write.rs @@ -230,7 +230,7 @@ impl CoreLoop { } // Index write-versions are published only once the write they // describe is durable. - if let Some(lsn) = task.wal_lsn() + if let Some(stamp) = self.task_write_stamp(task) && !touched.is_empty() { self.note_index_write_values( @@ -238,7 +238,7 @@ impl CoreLoop { crate::types::TenantId::new(tid), target_collection, &touched, - lsn, + stamp, ); } // The identity is the one INSERT minted: the declared primary diff --git a/nodedb/src/data/executor/handlers/upsert/exec/insert.rs b/nodedb/src/data/executor/handlers/upsert/exec/insert.rs index e929130e3..86ea23cbe 100644 --- a/nodedb/src/data/executor/handlers/upsert/exec/insert.rs +++ b/nodedb/src/data/executor/handlers/upsert/exec/insert.rs @@ -200,6 +200,22 @@ impl CoreLoop { return self.response_error(task, e); } + // Record the committed row's version and its touched index values, + // as a point put does. + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); + if let Some(stamp) = self.task_write_stamp(task) { + let mut tuples = std::mem::take(&mut prior.secondary_index_added); + tuples.append(&mut prior.secondary_index_removed); + tuples.append(&mut prior.bitemporal_index_tuples); + self.note_index_write_values( + task.request.database_id, + crate::types::TenantId::new(tid), + collection, + &tuples, + stamp, + ); + } + self.emit_put_event( task, tid, diff --git a/nodedb/src/data/executor/handlers/upsert/exec/overwrite.rs b/nodedb/src/data/executor/handlers/upsert/exec/overwrite.rs index 671958506..e3ff78b06 100644 --- a/nodedb/src/data/executor/handlers/upsert/exec/overwrite.rs +++ b/nodedb/src/data/executor/handlers/upsert/exec/overwrite.rs @@ -289,6 +289,22 @@ impl CoreLoop { return self.response_error(task, e); } + // Record the committed row's version and its touched index values, + // as a point put does. + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); + if let Some(stamp) = self.task_write_stamp(task) { + let mut tuples = std::mem::take(&mut outcome.secondary_index_added); + tuples.append(&mut outcome.secondary_index_removed); + tuples.append(&mut outcome.bitemporal_index_tuples); + self.note_index_write_values( + task.request.database_id, + crate::types::TenantId::new(tid), + collection, + &tuples, + stamp, + ); + } + // `current_bytes` is the pre-merge stored row, already read // above — thread it to the Event Plane as `old_value` so the // emitted WriteOp resolves to Update. diff --git a/nodedb/src/data/executor/handlers/vector.rs b/nodedb/src/data/executor/handlers/vector.rs index 2b180e7ea..ecff3f9eb 100644 --- a/nodedb/src/data/executor/handlers/vector.rs +++ b/nodedb/src/data/executor/handlers/vector.rs @@ -234,7 +234,7 @@ impl CoreLoop { // Record this write's version so cross-shard OCC read-set // validation (predicate reads always record the collection // floor) sees this insert. - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); self.response_ok(task) } Err(err) => self.response_error(task, err), @@ -364,7 +364,7 @@ impl CoreLoop { // surrogate (a superset of `execute_vector_delete`'s // node-id-scoped floor-only record below — correct and // more precise since the surrogate identity is known here). - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); } response } @@ -382,6 +382,7 @@ mod tests { use super::*; use crate::bridge::envelope::{PhysicalPlan, Priority, Request}; use crate::data::executor::core_loop::write_index::WriteKey; + use crate::data::executor::core_loop::write_index::tests::local; use crate::types::{Lsn, RequestId, TraceId, VShardId}; use nodedb_bridge::buffer::RingBuffer; use nodedb_physical::physical_plan::VectorOp; @@ -417,7 +418,7 @@ mod tests { } } - /// A task carrying `wal_lsn` so the handler's `note_*_write_lsn` calls + /// A task carrying `wal_lsn` so the handler's `note_*_write` calls /// (gated on `task.wal_lsn().is_some()`) actually fire, mirroring a live /// write dispatched with an allocated WAL LSN. fn make_task_with_lsn(lsn: u64) -> ExecutionTask { @@ -453,6 +454,7 @@ mod tests { wal_lsn: Some(Lsn::new(lsn)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), @@ -478,6 +480,7 @@ mod tests { assert_eq!(response.status, crate::bridge::envelope::Status::Ok); let key = WriteKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("docs"), @@ -486,20 +489,21 @@ mod tests { ), }; assert_eq!( - h.core.write_index.key_write_lsn(&key), - Some(Lsn::new(11)), + h.core.write_index.key_version(&key), + Some(local(11)), "vector insert must populate the per-key (surrogate) write-version index" ); let coll_key = crate::data::executor::core_loop::write_index::CollKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("docs"), }; assert_eq!( - h.core.write_index.collection_write_lsn(&coll_key), - Some(Lsn::new(11)), - "vector insert must advance the collection write-version floor \ + h.core.write_index.collection_version(&coll_key), + Some(local(11)), + "vector insert must advance the collection write version \ (predicate reads validate against the floor)" ); } diff --git a/nodedb/src/data/executor/handlers/vector_direct_resolve/apply.rs b/nodedb/src/data/executor/handlers/vector_direct_resolve/apply.rs index e85cbc0c0..0e72178bc 100644 --- a/nodedb/src/data/executor/handlers/vector_direct_resolve/apply.rs +++ b/nodedb/src/data/executor/handlers/vector_direct_resolve/apply.rs @@ -391,6 +391,7 @@ mod tests { wal_lsn: Some(Lsn::new(7)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/handlers/vector_direct_row.rs b/nodedb/src/data/executor/handlers/vector_direct_row.rs index 08428cb46..6e79cdd26 100644 --- a/nodedb/src/data/executor/handlers/vector_direct_row.rs +++ b/nodedb/src/data/executor/handlers/vector_direct_row.rs @@ -282,7 +282,7 @@ impl CoreLoop { } self.checkpoint_coordinator.mark_dirty("vector", 1); for surrogate in surrogates { - self.note_surrogate_write_lsn(task, tid, collection, surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); } } } diff --git a/nodedb/src/data/executor/handlers/vector_multi.rs b/nodedb/src/data/executor/handlers/vector_multi.rs index e6ea15310..e9df42590 100644 --- a/nodedb/src/data/executor/handlers/vector_multi.rs +++ b/nodedb/src/data/executor/handlers/vector_multi.rs @@ -122,7 +122,7 @@ impl CoreLoop { // Record this write's version keyed by the shared document surrogate // so cross-shard OCC read-set validation (predicate reads always // record the collection floor) sees this insert. - self.note_surrogate_write_lsn(task, tid, collection, document_surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, document_surrogate.as_u32()); match super::super::response_codec::encode_count("inserted_vectors", ids.len()) { Ok(bytes) => self.response_with_payload(task, bytes), @@ -161,7 +161,7 @@ impl CoreLoop { // Record this write's version keyed by the shared document // surrogate so cross-shard OCC read-set validation sees this // delete. - self.note_surrogate_write_lsn(task, tid, collection, document_surrogate.as_u32()); + self.note_surrogate_write(task, tid, collection, document_surrogate.as_u32()); } self.response_ok(task) } @@ -282,6 +282,7 @@ mod tests { Admission, ExemptReason, PhysicalPlan, Priority, Request, Status, }; use crate::data::executor::core_loop::CoreLoop; + use crate::data::executor::core_loop::write_index::tests::local; use crate::data::executor::core_loop::write_index::{CollKey, KeyRepr, WriteKey}; use crate::types::{DatabaseId, Lsn, ReadConsistency, RequestId, TenantId, TraceId, VShardId}; use nodedb_bridge::buffer::RingBuffer; @@ -317,7 +318,7 @@ mod tests { } } - /// A task carrying `wal_lsn` so `note_surrogate_write_lsn` (gated on + /// A task carrying `wal_lsn` so `note_surrogate_write` (gated on /// `task.wal_lsn().is_some()`) actually fires, mirroring a live write /// dispatched with an allocated WAL LSN. fn make_task_with_lsn(lsn: u64) -> ExecutionTask { @@ -353,6 +354,7 @@ mod tests { wal_lsn: Some(Lsn::new(lsn)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } @@ -376,26 +378,28 @@ mod tests { assert_eq!(response.status, Status::Ok); let key = WriteKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("chunks"), key: KeyRepr::Surrogate(document_surrogate.as_u32()), }; assert_eq!( - h.core.write_index.key_write_lsn(&key), - Some(Lsn::new(31)), + h.core.write_index.key_version(&key), + Some(local(31)), "multi-vector insert must populate the per-key (surrogate) write-version index" ); let coll_key = CollKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("chunks"), }; assert_eq!( - h.core.write_index.collection_write_lsn(&coll_key), - Some(Lsn::new(31)), - "multi-vector insert must advance the collection write-version floor" + h.core.write_index.collection_version(&coll_key), + Some(local(31)), + "multi-vector insert must advance the collection write version" ); } diff --git a/nodedb/src/data/executor/handlers/vector_sparse.rs b/nodedb/src/data/executor/handlers/vector_sparse.rs index 6e965c897..972ad5dbc 100644 --- a/nodedb/src/data/executor/handlers/vector_sparse.rs +++ b/nodedb/src/data/executor/handlers/vector_sparse.rs @@ -86,7 +86,7 @@ impl CoreLoop { self.checkpoint_coordinator.mark_dirty("vector", 1); // Sparse vectors are keyed by `doc_id: String` — no cross-engine // surrogate — so only the collection floor is recorded. - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); self.response_ok(task) } @@ -191,7 +191,7 @@ impl CoreLoop { self.checkpoint_coordinator.mark_dirty("vector", 1); // Sparse vectors are keyed by `doc_id: String` — no cross-engine // surrogate — so only the collection floor is recorded. - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); self.response_ok(task) } else { self.response_error(task, ErrorCode::NotFound) @@ -206,6 +206,7 @@ mod tests { Admission, ExemptReason, PhysicalPlan, Priority, Request, Status, }; use crate::data::executor::core_loop::CoreLoop; + use crate::data::executor::core_loop::write_index::tests::local; use crate::data::executor::core_loop::write_index::{CollKey, KeyRepr, WriteKey}; use crate::types::{DatabaseId, Lsn, ReadConsistency, RequestId, TenantId, TraceId, VShardId}; use nodedb_bridge::buffer::RingBuffer; @@ -242,7 +243,7 @@ mod tests { } } - /// A task carrying `wal_lsn` so `note_collection_write_lsn` (gated on + /// A task carrying `wal_lsn` so `note_collection_write` (gated on /// `task.wal_lsn().is_some()`) actually fires, mirroring a live write /// dispatched with an allocated WAL LSN. fn make_task_with_lsn(lsn: u64) -> ExecutionTask { @@ -278,6 +279,7 @@ mod tests { wal_lsn: Some(Lsn::new(lsn)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } @@ -298,26 +300,28 @@ mod tests { assert_eq!(response.status, Status::Ok); let coll_key = CollKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("sparse_docs"), }; assert_eq!( - h.core.write_index.collection_write_lsn(&coll_key), - Some(Lsn::new(41)), - "sparse insert must advance the collection write-version floor" + h.core.write_index.collection_version(&coll_key), + Some(local(41)), + "sparse insert must advance the collection write version" ); // Sparse vectors carry no cross-engine surrogate: no would-be // per-key `Surrogate` entry must exist for this write. let would_be_key = WriteKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("sparse_docs"), key: KeyRepr::Surrogate(0), }; assert_eq!( - h.core.write_index.key_write_lsn(&would_be_key), + h.core.write_index.key_version(&would_be_key), None, "sparse insert must not record a per-key surrogate entry" ); diff --git a/nodedb/src/data/executor/handlers/vector_upsert.rs b/nodedb/src/data/executor/handlers/vector_upsert.rs index 222b4b72e..39ac9763e 100644 --- a/nodedb/src/data/executor/handlers/vector_upsert.rs +++ b/nodedb/src/data/executor/handlers/vector_upsert.rs @@ -337,6 +337,7 @@ mod tests { Admission, ExemptReason, PhysicalPlan, Priority, Request, Status, }; use crate::data::executor::core_loop::CoreLoop; + use crate::data::executor::core_loop::write_index::tests::local; use crate::data::executor::core_loop::write_index::{CollKey, KeyRepr, WriteKey}; use crate::types::{DatabaseId, Lsn, ReadConsistency, RequestId, TenantId, TraceId, VShardId}; use nodedb_bridge::buffer::RingBuffer; @@ -373,7 +374,7 @@ mod tests { } } - /// A task carrying `wal_lsn` so `note_surrogate_write_lsn` (gated on + /// A task carrying `wal_lsn` so `note_surrogate_write` (gated on /// `task.wal_lsn().is_some()`) actually fires, mirroring a live write /// dispatched with an allocated WAL LSN. fn make_task_with_lsn(lsn: u64) -> ExecutionTask { @@ -409,6 +410,7 @@ mod tests { wal_lsn: Some(Lsn::new(lsn)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } @@ -460,26 +462,28 @@ mod tests { assert_eq!(response.status, Status::Ok); let key = WriteKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("primary_docs"), key: KeyRepr::Surrogate(surrogate.as_u32()), }; assert_eq!( - h.core.write_index.key_write_lsn(&key), - Some(Lsn::new(21)), + h.core.write_index.key_version(&key), + Some(local(21)), "direct upsert must populate the per-key (surrogate) write-version index" ); let coll_key = CollKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("primary_docs"), }; assert_eq!( - h.core.write_index.collection_write_lsn(&coll_key), - Some(Lsn::new(21)), - "direct upsert must advance the collection write-version floor" + h.core.write_index.collection_version(&coll_key), + Some(local(21)), + "direct upsert must advance the collection write version" ); } diff --git a/nodedb/src/data/executor/handlers/vector_write.rs b/nodedb/src/data/executor/handlers/vector_write.rs index 0b961b654..8abe344b2 100644 --- a/nodedb/src/data/executor/handlers/vector_write.rs +++ b/nodedb/src/data/executor/handlers/vector_write.rs @@ -67,10 +67,10 @@ impl CoreLoop { // validation sees this batch: per surrogate, a superset of the // collection floor. An empty batch records the floor only. for s in surrogates { - self.note_surrogate_write_lsn(task, tid, collection, s.as_u32()); + self.note_surrogate_write(task, tid, collection, s.as_u32()); } if surrogates.is_empty() { - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } match super::super::response_codec::encode_count("inserted", vectors.len()) { Ok(bytes) => self.response_with_payload(task, bytes), @@ -154,7 +154,7 @@ impl CoreLoop { // surrogate — recording it as a `KeyRepr::Surrogate` would be a // wrong identity in a different key space. Floor-only: a // predicate reader validates against the collection floor. - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); self.response_ok(task) } else { self.response_error(task, ErrorCode::NotFound) @@ -169,6 +169,7 @@ mod tests { Admission, ExemptReason, PhysicalPlan, Priority, Request, Status, }; use crate::data::executor::core_loop::CoreLoop; + use crate::data::executor::core_loop::write_index::tests::local; use crate::data::executor::core_loop::write_index::{CollKey, KeyRepr, WriteKey}; use crate::types::{Lsn, ReadConsistency, RequestId, TraceId, VShardId}; use nodedb_bridge::buffer::RingBuffer; @@ -205,7 +206,7 @@ mod tests { } } - /// A task carrying `wal_lsn` so `note_collection_write_lsn` (gated on + /// A task carrying `wal_lsn` so `note_collection_write` (gated on /// `task.wal_lsn().is_some()`) actually fires, mirroring a live write /// dispatched with an allocated WAL LSN. fn make_task_with_lsn(lsn: u64) -> ExecutionTask { @@ -241,6 +242,7 @@ mod tests { wal_lsn: Some(Lsn::new(lsn)), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } @@ -317,27 +319,29 @@ mod tests { assert_eq!(response.status, Status::Ok); let coll_key = CollKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("docs"), }; assert_eq!( - h.core.write_index.collection_write_lsn(&coll_key), - Some(Lsn::new(51)), - "vector delete must advance the collection write-version floor" + h.core.write_index.collection_version(&coll_key), + Some(local(51)), + "vector delete must advance the collection write version" ); // BRIGHT-LINE: `vector_id` (the internal HNSW node id) must never be // recorded as a `KeyRepr::Surrogate` — that is a different key space // from the cross-engine surrogate. let would_be_key = WriteKey { + vshard: VShardId::new(0), db: DatabaseId::DEFAULT, tenant: TenantId::new(1), collection: Box::from("docs"), key: KeyRepr::Surrogate(vector_id), }; assert_eq!( - h.core.write_index.key_write_lsn(&would_be_key), + h.core.write_index.key_version(&would_be_key), None, "vector delete must not record vector_id as a surrogate key" ); diff --git a/nodedb/src/data/executor/replay_task.rs b/nodedb/src/data/executor/replay_task.rs index 55ab326b5..a63b0b383 100644 --- a/nodedb/src/data/executor/replay_task.rs +++ b/nodedb/src/data/executor/replay_task.rs @@ -40,6 +40,7 @@ impl CoreLoop { wal_lsn, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/data/executor/task.rs b/nodedb/src/data/executor/task.rs index 88100dc68..4141eb8fc 100644 --- a/nodedb/src/data/executor/task.rs +++ b/nodedb/src/data/executor/task.rs @@ -17,7 +17,7 @@ pub struct ExecutionTask { /// WAL LSN the Control Plane allocated for this write at wal-dispatch time, /// carried alongside the request so the apply chokepoint can record the /// per-key / per-collection write version (see - /// [`CoreLoop::note_write_lsn`](crate::data::executor::core_loop::CoreLoop)). + /// [`CoreLoop::note_write`](crate::data::executor::core_loop::CoreLoop)). /// `None` for reads, control ops, and writes whose LSN is not (yet) threaded /// — the version index is skipped rather than advanced with a wrong value. /// Copied from [`Request::wal_lsn`](crate::bridge::envelope::Request) in @@ -140,6 +140,7 @@ mod tests { wal_lsn, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::Read, ), diff --git a/nodedb/src/data/executor/timeseries_checkpoint/flush.rs b/nodedb/src/data/executor/timeseries_checkpoint/flush.rs index e285dc949..444a71e06 100644 --- a/nodedb/src/data/executor/timeseries_checkpoint/flush.rs +++ b/nodedb/src/data/executor/timeseries_checkpoint/flush.rs @@ -164,6 +164,7 @@ mod tests { wal_lsn: wal_lsn.map(crate::types::Lsn::new), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Admitted, })) .expect("push request"); @@ -174,7 +175,7 @@ mod tests { /// Ingest one ILP line through the real dispatch path, exactly as the /// Control Plane does. `wal_lsn` is threaded BOTH on the op (the /// record the partition's replay stamp names) and on the request (what - /// `note_collection_write_lsn` raises the watermark from). + /// `note_collection_write` raises the watermark from). fn ingest(&mut self, host: &str, value: f64, ts_ms: i64, wal_lsn: u64) { let line = format!("{COLL},host={host} value={value} {}\n", ts_ms * 1_000_000); let r = self.send( diff --git a/nodedb/src/data/executor/timeseries_checkpoint/mod.rs b/nodedb/src/data/executor/timeseries_checkpoint/mod.rs index 299c7fdf0..091941263 100644 --- a/nodedb/src/data/executor/timeseries_checkpoint/mod.rs +++ b/nodedb/src/data/executor/timeseries_checkpoint/mod.rs @@ -6,7 +6,7 @@ //! //! `columnar_memtables` holds one `ColumnarMemtable` per timeseries collection, //! and every ILP / JSON / msgpack ingest lands there. Those rows advance the -//! core watermark (`execute_timeseries_ingest` calls `note_collection_write_lsn` +//! core watermark (`execute_timeseries_ingest` calls `note_collection_write` //! with the Control Plane's `wal_lsn`). The periodic checkpoint reports them as //! durable, and the manager then removes the `TimeseriesBatch` records below //! that LSN. The checkpoint must therefore flush every memtable first. The diff --git a/nodedb/src/data/executor/wal_replay/crdt.rs b/nodedb/src/data/executor/wal_replay/crdt.rs index 17c75acbd..e072d9383 100644 --- a/nodedb/src/data/executor/wal_replay/crdt.rs +++ b/nodedb/src/data/executor/wal_replay/crdt.rs @@ -401,7 +401,7 @@ impl CoreLoop { // Every successfully imported payload changed authoritative Loro // state, including a snapshot import that projects no row. Record // its exact durable collection floor either way. - self.note_replay_write_lsn( + self.note_replay_write( record.header.database_id, tid.as_u64(), collection, @@ -1054,15 +1054,20 @@ mod crdt_replay_tests { "matching fenced retry must rebuild its sparse projection" ); assert_eq!( - h.core.write_index.collection_write_lsn( + h.core.write_index.collection_version( &crate::data::executor::core_loop::write_index::CollKey { + vshard: crate::data::executor::core_loop::write_index::tests::replay_home( + db, collection + ), db, tenant: tid, collection: Box::from(collection), } ), - Some(crate::types::Lsn::new(3)), - "only the correctly fenced retry advances the replay write floor" + Some(crate::data::executor::core_loop::write_index::tests::local( + 3 + )), + "only the correctly fenced retry advances the replay write version" ); } @@ -1233,14 +1238,23 @@ mod crdt_replay_tests { h } + /// The replayed version of `users`, as the WAL LSN it was written at. fn users_write_lsn(h: &CoreHarness, tid: TenantId) -> Option { - h.core.write_index.collection_write_lsn( + use crate::data::executor::core_loop::write_index::tests::{local, replay_home}; + let version = h.core.write_index.collection_version( &crate::data::executor::core_loop::write_index::CollKey { + vshard: replay_home(DatabaseId::DEFAULT, "users"), db: DatabaseId::DEFAULT, tenant: tid, collection: Box::from("users"), }, - ) + )?; + assert_eq!( + version, + local(version.local), + "a single-node replay records a local version" + ); + Some(crate::types::Lsn::new(version.local)) } /// A replayed rejection the full queue refuses stops replay at its diff --git a/nodedb/src/data/executor/wal_replay/kv.rs b/nodedb/src/data/executor/wal_replay/kv.rs index 4364848d4..a5c826bc1 100644 --- a/nodedb/src/data/executor/wal_replay/kv.rs +++ b/nodedb/src/data/executor/wal_replay/kv.rs @@ -303,7 +303,7 @@ impl CoreLoop { self.kv_engine .delete(database_id, tenant_id, &collection, &keys, now_ms); for deleted_key in &keys { - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, @@ -336,13 +336,7 @@ impl CoreLoop { }]); } self.kv_engine.truncate(database_id, tenant_id, &collection); - self.note_replay_write_lsn( - database_id, - tenant_id, - &collection, - None, - record_lsn, - ); + self.note_replay_write(database_id, tenant_id, &collection, None, record_lsn); deletes += 1; continue; } diff --git a/nodedb/src/data/executor/wal_replay/kv_put.rs b/nodedb/src/data/executor/wal_replay/kv_put.rs index ef1ed814f..4b4c97c80 100644 --- a/nodedb/src/data/executor/wal_replay/kv_put.rs +++ b/nodedb/src/data/executor/wal_replay/kv_put.rs @@ -119,7 +119,7 @@ impl CoreLoop { self.replay_record_unapplied("kv", "put_identity", record_lsn, &e.to_string()); return Some(0); } - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, @@ -176,7 +176,7 @@ impl CoreLoop { }, expire_at_ms, ); - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, @@ -268,7 +268,7 @@ impl CoreLoop { return Some(0); } for (entry_key, _entry_value) in &entries { - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, diff --git a/nodedb/src/data/executor/wal_replay_all.rs b/nodedb/src/data/executor/wal_replay_all.rs index 258edc698..505ac6d2b 100644 --- a/nodedb/src/data/executor/wal_replay_all.rs +++ b/nodedb/src/data/executor/wal_replay_all.rs @@ -29,6 +29,26 @@ use super::core_loop::CoreLoop; use super::core_loop::fail_stop::FailStopCause; impl CoreLoop { + /// Bound the versions restart replay could not rebuild, then forget the + /// record stamps. + /// + /// A record an engine checkpoint already held applies no write, so its + /// keys keep no version. Each vShard this core owns therefore holds every + /// write through the newest of its WAL records, and a read older than + /// that no longer validates against a row with no recorded version. + fn settle_replayed_versions(&mut self, num_cores: usize) { + let owned: Vec<_> = self + .write_index + .record_stamps + .latest_by_vshard() + .filter(|(vshard, _)| crate::types::core_for_vshard(*vshard, num_cores) == self.core_id) + .collect(); + for (vshard, version) in owned { + self.write_index.raise_installed(vshard, version); + } + self.write_index.record_stamps.clear(); + } + /// Replay every WAL record class into this core's engines, in the exact /// order restart correctness requires. No-op when `records` is empty. /// @@ -59,6 +79,12 @@ impl CoreLoop { .seed_replayed_through(crate::types::Lsn::new(wal_end)); } + // Every write a replay arm applies takes its record's vShard and the + // log position of the data-group entry the record applies, so a + // recovered write records the version its live apply did. + self.write_index.record_stamps = + super::core_loop::write_index::RecordStamps::from_wal(records); + crate::fail_point!("replay::before_engine_passes"); // Every engine-bearing record class — standalone (autocommit) records @@ -71,7 +97,9 @@ impl CoreLoop { // Fatal on error: a redo group that cannot be reconstituted is a // committed transaction that cannot be applied, and continuing would // open the database with a hole in the replayed suffix. - if let Err(e) = self.replay_engines_in_lsn_order(records, num_cores, tombstones) { + let replayed = self.replay_engines_in_lsn_order(records, num_cores, tombstones); + self.settle_replayed_versions(num_cores); + if let Err(e) = replayed { error!( core_id, error = %e, @@ -160,6 +188,135 @@ mod tests { .expect("build record") } + /// A KV put record the apply of the proposal keyed `apply_key` appended. + fn keyed_kv_put_record(lsn: u64, key: &[u8], apply_key: u64) -> WalRecord { + WalRecord::new_stamped( + WalRecordArgs { + record_type: RecordType::Put as u32, + lsn, + tenant_id: 1, + vshard_id: 0, + database_id: 0, + payload: zerompk::to_msgpack_vec(&( + "kv_put", + "cache", + key.to_vec(), + b"v".to_vec(), + 0u64, + None::, + 1u32, + )) + .expect("encode kv put"), + encryption_key: None, + preamble_bytes: None, + }, + nodedb_wal::record::RecordStamp { + apply_key, + event_source: nodedb_wal::NO_EVENT_SOURCE, + commit_hlc: 0, + }, + ) + .expect("build record") + } + + /// The `ChangePosition` marker of the entry at `(epoch, log_index)` whose + /// proposal key is `apply_key`. + fn position_marker(lsn: u64, apply_key: u64, epoch: u64, log_index: u64) -> WalRecord { + let marker = crate::event::cdc::position::ChangePositionMarker { + apply_key, + position: crate::event::cdc::position::ReplicatedPosition { + epoch, + group_id: 9, + log_index, + }, + }; + WalRecord::new(WalRecordArgs { + record_type: RecordType::ChangePosition as u32, + lsn, + tenant_id: 1, + vshard_id: 0, + database_id: 0, + payload: marker.to_bytes().to_vec(), + encryption_key: None, + preamble_bytes: None, + }) + .expect("build marker") + } + + fn kv_version(core: &CoreLoop, key: &[u8]) -> Option { + core.write_index + .key_version(&crate::data::executor::core_loop::write_index::WriteKey { + vshard: crate::types::VShardId::new(0), + db: DatabaseId::DEFAULT, + tenant: TenantId::new(TID), + collection: Box::from("cache"), + key: crate::types::KeyRepr::KvKey(Box::from(key)), + }) + } + + /// Restart replay rebuilds each write's version from the data-group + /// entry its record applied, the version every replica recorded live. + /// A record no entry applied takes its WAL LSN, as it did live. + #[test] + fn replay_records_the_entry_position_each_record_applied() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _req, _resp) = make_core_with_dir(dir.path()); + let records = vec![ + position_marker(10, 0xA1, 2, 70), + keyed_kv_put_record(11, b"replicated", 0xA1), + keyed_kv_put_record(12, b"local", 0), + ]; + + core.replay_all_wal(&records, 1, &TombstoneSet::new()) + .expect("replay"); + + let entry = nodedb_types::WriteVersion::logged(2, 70); + assert_eq!(kv_version(&core, b"replicated"), Some(entry)); + assert_eq!( + kv_version(&core, b"local"), + Some(nodedb_types::WriteVersion::local_after(entry, 12)) + ); + assert_eq!( + core.write_index.latest(crate::types::VShardId::new(0)), + nodedb_types::WriteVersion::local_after(entry, 12) + ); + assert!( + core.write_index.record_stamps.get(11).is_none(), + "replay forgets the record stamps once it ends" + ); + } + + /// A key whose record an engine checkpoint already held keeps no version + /// after replay. A read older than the WAL's newest record then no longer + /// validates against it. + #[test] + fn replay_bounds_the_versions_a_checkpoint_hid() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _req, _resp) = make_core_with_dir(dir.path()); + let records = vec![ + position_marker(20, 0xB2, 0, 40), + keyed_kv_put_record(21, b"k", 0xB2), + ]; + core.replay_all_wal(&records, 1, &TombstoneSet::new()) + .expect("replay"); + + let never_replayed = nodedb_types::calvin::ReadKeyIdent::Point( + crate::types::KeyRepr::KvKey(Box::from(b"absent".as_slice())), + ); + let valid = |read| { + core.write_index.read_is_valid( + DatabaseId::DEFAULT, + TenantId::new(TID), + crate::types::VShardId::new(0), + "cache", + &never_replayed, + nodedb_types::WriteVersion::logged(0, read), + ) + }; + assert!(!valid(39), "a read older than the WAL's newest record"); + assert!(valid(40), "a read at the WAL's newest record"); + } + /// Every record replay reads is decided when replay ends, so a checkpoint /// written afterwards names all of them through its prefix, and the /// applied set holds none of them. diff --git a/nodedb/src/data/executor/wal_replay_columnar_image.rs b/nodedb/src/data/executor/wal_replay_columnar_image.rs index 075537626..9902d4821 100644 --- a/nodedb/src/data/executor/wal_replay_columnar_image.rs +++ b/nodedb/src/data/executor/wal_replay_columnar_image.rs @@ -329,7 +329,7 @@ impl CoreLoop { collection, ); self.checkpoint_coordinator.mark_dirty("columnar", written); - self.note_collection_write_lsn(task, collection); + self.note_collection_write(task, collection); } Ok(written) } diff --git a/nodedb/src/data/executor/wal_replay_fts.rs b/nodedb/src/data/executor/wal_replay_fts.rs index 6dc434f35..175e83f26 100644 --- a/nodedb/src/data/executor/wal_replay_fts.rs +++ b/nodedb/src/data/executor/wal_replay_fts.rs @@ -77,6 +77,7 @@ impl CoreLoop { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/data/executor/wal_replay_kv_atomic.rs b/nodedb/src/data/executor/wal_replay_kv_atomic.rs index 7b3d64699..15db591b1 100644 --- a/nodedb/src/data/executor/wal_replay_kv_atomic.rs +++ b/nodedb/src/data/executor/wal_replay_kv_atomic.rs @@ -122,7 +122,7 @@ impl CoreLoop { } }; if swapped { - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, @@ -180,7 +180,7 @@ impl CoreLoop { &gate, ) { Ok(_) => { - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, @@ -313,7 +313,7 @@ impl CoreLoop { self.replay_swap_error("getset", &collection, &key, record_lsn, error); return Some(0); } - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, diff --git a/nodedb/src/data/executor/wal_replay_kv_expiry.rs b/nodedb/src/data/executor/wal_replay_kv_expiry.rs index 8c569b995..403e31192 100644 --- a/nodedb/src/data/executor/wal_replay_kv_expiry.rs +++ b/nodedb/src/data/executor/wal_replay_kv_expiry.rs @@ -86,7 +86,7 @@ impl CoreLoop { ); return Some(0); } - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, @@ -139,7 +139,7 @@ impl CoreLoop { ); return Some(0); } - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, diff --git a/nodedb/src/data/executor/wal_replay_kv_field.rs b/nodedb/src/data/executor/wal_replay_kv_field.rs index ef0a6e167..34be24d81 100644 --- a/nodedb/src/data/executor/wal_replay_kv_field.rs +++ b/nodedb/src/data/executor/wal_replay_kv_field.rs @@ -90,7 +90,7 @@ impl CoreLoop { self.replay_record_unapplied("kv", "field_set_identity", record_lsn, &e.to_string()); return Some(0); } - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, diff --git a/nodedb/src/data/executor/wal_replay_kv_incr.rs b/nodedb/src/data/executor/wal_replay_kv_incr.rs index 5a53d7453..ad294bdb7 100644 --- a/nodedb/src/data/executor/wal_replay_kv_incr.rs +++ b/nodedb/src/data/executor/wal_replay_kv_incr.rs @@ -99,7 +99,7 @@ impl CoreLoop { }; let applied = self.log_kv_incr_result(&collection, &key, delta, record_lsn, result); if applied > 0 { - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, diff --git a/nodedb/src/data/executor/wal_replay_kv_insert_conflict.rs b/nodedb/src/data/executor/wal_replay_kv_insert_conflict.rs index b989b4f0f..3a4eec607 100644 --- a/nodedb/src/data/executor/wal_replay_kv_insert_conflict.rs +++ b/nodedb/src/data/executor/wal_replay_kv_insert_conflict.rs @@ -172,7 +172,7 @@ impl CoreLoop { ); return 0; } - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, collection, diff --git a/nodedb/src/data/executor/wal_replay_kv_predicate.rs b/nodedb/src/data/executor/wal_replay_kv_predicate.rs index 4fe215c5d..b8a672a26 100644 --- a/nodedb/src/data/executor/wal_replay_kv_predicate.rs +++ b/nodedb/src/data/executor/wal_replay_kv_predicate.rs @@ -92,7 +92,7 @@ impl CoreLoop { } // The key set is resolved at replay time rather than carried, so the // collection floor is the version this can record. - self.note_replay_write_lsn(database_id, tenant_id, &collection, None, record_lsn); + self.note_replay_write(database_id, tenant_id, &collection, None, record_lsn); Some(written) } @@ -138,7 +138,7 @@ impl CoreLoop { let removed = self .kv_engine .delete(database_id, tenant_id, &collection, &keys, now_ms); - self.note_replay_write_lsn(database_id, tenant_id, &collection, None, record_lsn); + self.note_replay_write(database_id, tenant_id, &collection, None, record_lsn); Some(removed) } } diff --git a/nodedb/src/data/executor/wal_replay_kv_transfer.rs b/nodedb/src/data/executor/wal_replay_kv_transfer.rs index ec0ae0eac..8b8b8183f 100644 --- a/nodedb/src/data/executor/wal_replay_kv_transfer.rs +++ b/nodedb/src/data/executor/wal_replay_kv_transfer.rs @@ -181,14 +181,14 @@ impl CoreLoop { return 0; } } - self.note_replay_write_lsn( + self.note_replay_write( p.database_id, p.tenant_id, p.collection, Some(KeyRepr::KvKey(Box::from(p.source_key))), p.record_lsn, ); - self.note_replay_write_lsn( + self.note_replay_write( p.database_id, p.tenant_id, p.collection, @@ -282,14 +282,14 @@ impl CoreLoop { return (0, 0); } - self.note_replay_write_lsn( + self.note_replay_write( p.database_id, p.tenant_id, p.source_collection, Some(KeyRepr::KvKey(Box::from(p.item_key))), p.record_lsn, ); - self.note_replay_write_lsn( + self.note_replay_write( p.database_id, p.tenant_id, p.dest_collection, diff --git a/nodedb/src/data/executor/wal_replay_kv_ttl.rs b/nodedb/src/data/executor/wal_replay_kv_ttl.rs index 91fcbdfab..251ba7ae0 100644 --- a/nodedb/src/data/executor/wal_replay_kv_ttl.rs +++ b/nodedb/src/data/executor/wal_replay_kv_ttl.rs @@ -277,6 +277,7 @@ mod tests { wal_lsn: None, resolved_now_ms, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/wal_replay_redo_document.rs b/nodedb/src/data/executor/wal_replay_redo_document.rs index 20d2e70ae..725a2908b 100644 --- a/nodedb/src/data/executor/wal_replay_redo_document.rs +++ b/nodedb/src/data/executor/wal_replay_redo_document.rs @@ -258,7 +258,7 @@ impl CoreLoop { } if applied { puts += 1; - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, @@ -351,7 +351,7 @@ impl CoreLoop { } if removed { deletes += 1; - self.note_replay_write_lsn( + self.note_replay_write( database_id, tenant_id, &collection, @@ -401,7 +401,7 @@ fn is_kv_put_record(payload: &[u8]) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::types::{DatabaseId, Lsn, TenantId}; + use crate::types::{DatabaseId, TenantId}; use crate::wal::{RedoRecord, RedoSubRecord}; use nodedb_types::Surrogate; use nodedb_wal::WalRecord; @@ -721,6 +721,7 @@ mod tests { #[test] fn redo_replay_populates_write_version_index() { + use crate::data::executor::core_loop::write_index::tests::{local, replay_home}; use crate::data::executor::core_loop::write_index::{CollKey, WriteKey}; let mut h = make_core(); @@ -747,49 +748,53 @@ mod tests { let tenant = TenantId::new(7); let doc_key = WriteKey { + vshard: replay_home(db, "notes"), db, tenant, collection: Box::from("notes"), key: KeyRepr::Surrogate(surrogate), }; assert_eq!( - h.core.write_index.key_write_lsn(&doc_key), - Some(Lsn::new(1)), + h.core.write_index.key_version(&doc_key), + Some(local(1)), "document redo put must populate the per-key write-version index" ); let doc_coll_key = CollKey { + vshard: replay_home(db, "notes"), db, tenant, collection: Box::from("notes"), }; assert_eq!( - h.core.write_index.collection_write_lsn(&doc_coll_key), - Some(Lsn::new(1)), - "document redo put must advance the collection write-version floor" + h.core.write_index.collection_version(&doc_coll_key), + Some(local(1)), + "document redo put must advance the collection write version" ); let kv_key = WriteKey { + vshard: replay_home(db, "sessions"), db, tenant, collection: Box::from("sessions"), key: KeyRepr::KvKey(Box::from(b"s1".as_slice())), }; assert_eq!( - h.core.write_index.key_write_lsn(&kv_key), - Some(Lsn::new(1)), + h.core.write_index.key_version(&kv_key), + Some(local(1)), "kv redo put must populate the per-key write-version index" ); let kv_coll_key = CollKey { + vshard: replay_home(db, "sessions"), db, tenant, collection: Box::from("sessions"), }; assert_eq!( - h.core.write_index.collection_write_lsn(&kv_coll_key), - Some(Lsn::new(1)), - "kv redo put must advance the collection write-version floor" + h.core.write_index.collection_version(&kv_coll_key), + Some(local(1)), + "kv redo put must advance the collection write version" ); // A later delete at a higher LSN must bump the document key's write @@ -827,8 +832,8 @@ mod tests { .expect("redo replay must succeed"); assert_eq!( - h.core.write_index.key_write_lsn(&doc_key), - Some(Lsn::new(2)), + h.core.write_index.key_version(&doc_key), + Some(local(2)), "redo delete must bump the key's write-version above the prior put" ); } diff --git a/nodedb/src/data/executor/wal_replay_redo_graph.rs b/nodedb/src/data/executor/wal_replay_redo_graph.rs index 9820ff67b..fdb8ae4ca 100644 --- a/nodedb/src/data/executor/wal_replay_redo_graph.rs +++ b/nodedb/src/data/executor/wal_replay_redo_graph.rs @@ -442,6 +442,7 @@ impl CoreLoop { wal_lsn, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), @@ -582,7 +583,7 @@ mod tests { } #[test] - fn redo_graph_edge_put_records_write_version_floor() { + fn redo_graph_edge_put_records_its_write_version() { use crate::data::executor::core_loop::write_index::{KeyRepr, WriteKey}; let mut h = make_core(); @@ -596,7 +597,10 @@ mod tests { ) .expect("redo replay must succeed"); + // The edge applies on the vShard its record names, at the record's + // LSN: no registered stamp names an entry for it. let write_key = WriteKey { + vshard: crate::types::VShardId::new(record.header.vshard_id), db: DatabaseId::new(0), tenant: crate::types::TenantId::new(7), collection: Box::from("knows"), @@ -607,9 +611,11 @@ mod tests { }, }; assert_eq!( - h.core.write_index.key_write_lsn(&write_key), - Some(Lsn::new(1)), - "graph edge redo replay must record the write-version floor at the record's LSN" + h.core.write_index.key_version(&write_key), + Some(crate::data::executor::core_loop::write_index::tests::local( + 1 + )), + "graph edge redo replay must record the write version at the record's LSN" ); } diff --git a/nodedb/src/data/executor/wal_replay_redo_graph_cut.rs b/nodedb/src/data/executor/wal_replay_redo_graph_cut.rs index 7b025fbb5..c47a406e4 100644 --- a/nodedb/src/data/executor/wal_replay_redo_graph_cut.rs +++ b/nodedb/src/data/executor/wal_replay_redo_graph_cut.rs @@ -153,6 +153,7 @@ mod tests { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Exempt(ExemptReason::Read), }) } diff --git a/nodedb/src/data/executor/wal_replay_row_image_tests.rs b/nodedb/src/data/executor/wal_replay_row_image_tests.rs index 7f70009ae..9be069945 100644 --- a/nodedb/src/data/executor/wal_replay_row_image_tests.rs +++ b/nodedb/src/data/executor/wal_replay_row_image_tests.rs @@ -119,6 +119,7 @@ fn request(plan: PhysicalPlan, wal_lsn: Option) -> Request { wal_lsn, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Admitted, } } diff --git a/nodedb/src/data/executor/wal_replay_spatial.rs b/nodedb/src/data/executor/wal_replay_spatial.rs index cb24fc978..e6c9dc85c 100644 --- a/nodedb/src/data/executor/wal_replay_spatial.rs +++ b/nodedb/src/data/executor/wal_replay_spatial.rs @@ -105,6 +105,7 @@ impl CoreLoop { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/data/executor/wal_replay_vector_task.rs b/nodedb/src/data/executor/wal_replay_vector_task.rs index 3f6995c45..4cce0934f 100644 --- a/nodedb/src/data/executor/wal_replay_vector_task.rs +++ b/nodedb/src/data/executor/wal_replay_vector_task.rs @@ -51,6 +51,7 @@ impl CoreLoop { wal_lsn, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: crate::bridge::envelope::Admission::Exempt( crate::bridge::envelope::ExemptReason::AlreadyOrdered, ), diff --git a/nodedb/src/data/executor/write_set_crash_tests.rs b/nodedb/src/data/executor/write_set_crash_tests.rs index d3615b2d3..bcfa36b8c 100644 --- a/nodedb/src/data/executor/write_set_crash_tests.rs +++ b/nodedb/src/data/executor/write_set_crash_tests.rs @@ -197,6 +197,7 @@ fn request(plan: PhysicalPlan, origin: GroupOrigin) -> Request { wal_lsn: Some(origin.lsn), resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: Admission::Admitted, } } diff --git a/nodedb/src/data/runtime/event_loop.rs b/nodedb/src/data/runtime/event_loop.rs index 5fbba58f2..35b7b03a1 100644 --- a/nodedb/src/data/runtime/event_loop.rs +++ b/nodedb/src/data/runtime/event_loop.rs @@ -202,7 +202,7 @@ fn drain_and_reject(core: &mut CoreLoop, core_id: usize) { detail: format!("core-{core_id} is degraded after repeated panics"), })), stage_vote: None, - read_version_lsn: crate::types::Lsn::ZERO, + read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }; if let Err(e) = core diff --git a/nodedb/src/engine/array/store/catalog/scan.rs b/nodedb/src/engine/array/store/catalog/scan.rs index 9289ad5f3..d44637572 100644 --- a/nodedb/src/engine/array/store/catalog/scan.rs +++ b/nodedb/src/engine/array/store/catalog/scan.rs @@ -208,15 +208,18 @@ impl ArrayStore { valid_at_ms: Option, ) -> nodedb_array::ArrayResult { use nodedb_array::query::ceiling::CeilingParams; - // Find the hilbert_prefix for this coord. - let hilbert_prefix = { - use nodedb_array::tile::tile_id_for_cell; - let tile = tile_id_for_cell(&self.schema, coord, 0).map_err(|e| { - nodedb_array::ArrayError::SegmentCorruption { + // Find the hilbert_prefix for this coord. A coordinate outside the + // domain has no tile: a cell write refuses it, so no cell is there. + let hilbert_prefix = match nodedb_array::tile::tile_id_for_cell(&self.schema, coord, 0) { + Ok(tile) => tile.hilbert_prefix, + Err(nodedb_array::ArrayError::CoordOutOfDomain { .. }) => { + return Ok(CeilingResult::NotFound); + } + Err(e) => { + return Err(nodedb_array::ArrayError::SegmentCorruption { detail: format!("ceiling_for_coord: tile id: {e}"), - } - })?; - tile.hilbert_prefix + }); + } }; let versions = self.cell_versions_for_coord(hilbert_prefix, coord, system_as_of)?; let params = CeilingParams { diff --git a/nodedb/src/types/mod.rs b/nodedb/src/types/mod.rs index 8ef167f00..bc6d44077 100644 --- a/nodedb/src/types/mod.rs +++ b/nodedb/src/types/mod.rs @@ -5,6 +5,7 @@ pub mod core_route; pub mod hash_chain; pub mod id; pub mod lsn; +pub mod read_versions; pub mod record_home; pub mod replay_stamp; pub mod snapshot; @@ -15,6 +16,7 @@ pub use core_route::core_for_vshard; pub use id::{DatabaseId, DocumentId, RequestId, TenantId, TxnId, VShardId}; pub use lsn::Lsn; pub use nodedb_types::{KeyRepr, SpanId, TraceId}; +pub use read_versions::ReadVersions; pub use record_home::{HomedRecord, RecordHomes}; pub use snapshot::{ ArrayCellsBlob, SurrogateBindEntry, TenantDataSnapshot, TsFlushedCollectionBlob, diff --git a/nodedb/src/types/read_versions.rs b/nodedb/src/types/read_versions.rs new file mode 100644 index 000000000..3a6d9e0e0 --- /dev/null +++ b/nodedb/src/types/read_versions.rs @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The write versions one read observed, one per vShard it read. + +use smallvec::SmallVec; + +use nodedb_types::{ShardVersion, WriteVersion}; + +use super::VShardId; + +/// The versions one read observed, one per vShard. +/// +/// A version positions a write within its vShard only, so each one travels +/// with the vShard it belongs to. A read of one collection observes one +/// vShard, so the common case holds one version without allocating. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ReadVersions(SmallVec<[ShardVersion; 1]>); + +impl ReadVersions { + /// No versions: the response observed nothing a read validates. + pub fn new() -> Self { + Self::default() + } + + /// One vShard observed at `version`. + pub fn single(vshard: VShardId, version: WriteVersion) -> Self { + let mut versions = Self::new(); + versions.note(vshard, version); + versions + } + + /// Note that the read observed `vshard` at `version`. + /// + /// Two answers for one vShard keep the higher. Only the core that holds a + /// vShard records versions of it, so every other answer is lower. + pub fn note(&mut self, vshard: VShardId, version: WriteVersion) { + self.note_raw(vshard.as_u32(), version); + } + + fn note_raw(&mut self, vshard: u32, version: WriteVersion) { + match self.0.iter_mut().find(|shard| shard.vshard == vshard) { + Some(shard) => shard.version = shard.version.max(version), + None => self.0.push(ShardVersion { vshard, version }), + } + } + + /// Fold `other` in, keeping the higher version per vShard. + pub fn merge(&mut self, other: &ReadVersions) { + for shard in &other.0 { + self.note_raw(shard.vshard, shard.version); + } + } + + /// The version the read observed on `vshard`, if it reported one. + pub fn of(&self, vshard: VShardId) -> Option { + self.0 + .iter() + .find(|shard| shard.vshard == vshard.as_u32()) + .map(|shard| shard.version) + } + + /// Every observed vShard, in the order first noted. + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// The versions as they travel between nodes. + pub fn to_wire(&self) -> Vec { + self.0.to_vec() + } + + /// The versions a node sent. + pub fn from_wire(versions: &[ShardVersion]) -> Self { + let mut read = Self::new(); + for shard in versions { + read.note_raw(shard.vshard, shard.version); + } + read + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_vshard_noted_twice_keeps_the_higher_version() { + let mut versions = ReadVersions::single(VShardId::new(3), WriteVersion::logged(0, 9)); + versions.note(VShardId::new(3), WriteVersion::logged(0, 4)); + versions.note(VShardId::new(5), WriteVersion::logged(1, 2)); + assert_eq!( + versions.of(VShardId::new(3)), + Some(WriteVersion::logged(0, 9)) + ); + assert_eq!( + versions.of(VShardId::new(5)), + Some(WriteVersion::logged(1, 2)) + ); + assert_eq!(versions.of(VShardId::new(6)), None); + } + + #[test] + fn versions_survive_the_wire() { + let mut versions = ReadVersions::single(VShardId::new(3), WriteVersion::logged(0, 9)); + versions.merge(&ReadVersions::single( + VShardId::new(8), + WriteVersion::logged(2, 1), + )); + assert_eq!(ReadVersions::from_wire(&versions.to_wire()), versions); + } +} diff --git a/nodedb/tests/crash_harness/vshards.rs b/nodedb/tests/crash_harness/vshards.rs index e3b55cf92..f9187388f 100644 --- a/nodedb/tests/crash_harness/vshards.rs +++ b/nodedb/tests/crash_harness/vshards.rs @@ -40,19 +40,20 @@ pub fn single_node_data_group(name: &str) -> u32 { /// The data Raft groups a single-node server applies the cell writes of one /// `INT64` dimension `[lo..hi]` array in. /// -/// An array cell write routes by its coordinate, never by the array's name. -/// The coordinate's Hilbert prefix picks a bucket from its top `prefix_bits` -/// bits, and the bucket owns vShard `bucket * (1024 >> prefix_bits)`. At the -/// default of 8 bits every array vShard is a multiple of 4, so every cell of -/// every array applies in data group 1. Only `prefix_bits` of 9 or more -/// spreads cells over more than one group. +/// An array cell write routes by its tile, never by the array's name. The +/// tile's key picks a bucket from its top `prefix_bits` bits, and the bucket +/// owns vShard `bucket * (1024 >> prefix_bits)`. At the default of 8 bits +/// every array vShard is a multiple of 4, so every cell of every array +/// applies in data group 1. Only `prefix_bits` of 9 or more spreads cells +/// over more than one group. `tile_extent` must match the array's +/// `TILE_EXTENTS`. pub struct ArrayCellRoute { schema: nodedb_array::ArraySchema, prefix_bits: u8, } impl ArrayCellRoute { - pub fn new(lo: i64, hi: i64, prefix_bits: u8) -> Self { + pub fn new(lo: i64, hi: i64, tile_extent: u64, prefix_bits: u8) -> Self { use nodedb_array::schema::{ArraySchemaBuilder, AttrSpec, AttrType, DimSpec, DimType}; use nodedb_array::types::domain::{Domain, DomainBound}; let schema = ArraySchemaBuilder::new("route") @@ -62,7 +63,7 @@ impl ArrayCellRoute { Domain::new(DomainBound::Int64(lo), DomainBound::Int64(hi)), )) .attr(AttrSpec::new("v", AttrType::Float64, true)) - .tile_extents(vec![1]) + .tile_extents(vec![tile_extent]) .build() .expect("a one-dimension routing schema"); Self { @@ -74,7 +75,7 @@ impl ArrayCellRoute { /// The data group the cell at `coord` applies in. pub fn group(&self, coord: i64) -> u32 { use nodedb_array::types::coord::value::CoordValue; - let prefix = nodedb_array::encode_hilbert_prefix(&self.schema, &[CoordValue::Int64(coord)]) + let prefix = nodedb_array::cell_tile_prefix(&self.schema, &[CoordValue::Int64(coord)]) .unwrap_or_else(|e| panic!("coordinate {coord} outside the routing schema: {e}")); let vshard = nodedb_cluster::distributed_array::array_vshard_for_tile(prefix, self.prefix_bits) diff --git a/nodedb/tests/crash_replay_stamp/engines.rs b/nodedb/tests/crash_replay_stamp/engines.rs index d617552df..bd04a1c81 100644 --- a/nodedb/tests/crash_replay_stamp/engines.rs +++ b/nodedb/tests/crash_replay_stamp/engines.rs @@ -63,9 +63,9 @@ async fn a_columnar_write_in_flight_at_a_checkpoint_survives_kill_9() { /// That checkpoint flushes the held array with a stamp naming B writes above /// A. Boot 3 replays A below that stamp's highest LSN (`in_flight`). /// -/// An array cell write routes by its coordinate, not by the array's name (see -/// [`ArrayCellRoute`]). Both arrays take `prefix_bits = 10`, so their cells -/// spread over every data group. Each B cell sits at a coordinate whose group +/// An array cell write routes by its tile, not by the array's name (see +/// [`ArrayCellRoute`]). Both arrays take `prefix_bits = 10`, so the tiles of +/// the applied array spread over every data group. Each B cell sits at a coordinate whose group /// is not the group of A's cell. #[tokio::test(flavor = "multi_thread")] async fn an_array_write_in_flight_at_a_checkpoint_survives_kill_9() { @@ -107,10 +107,11 @@ const ARRAY_PREFIX_BITS: u8 = 10; /// The coordinate of B number `n` in the array case: the `n`th coordinate of /// `[0..1023]` whose cell applies in another data group than A's cell, coord 1 -/// of `[0..15]`. The domains match the case's `CREATE ARRAY` statements. +/// of `[0..15]`. The domains and tile extents match the case's `CREATE +/// ARRAY` statements. fn applied_array_coord(n: usize) -> i64 { - let held_group = ArrayCellRoute::new(0, 15, ARRAY_PREFIX_BITS).group(1); - let applied = ArrayCellRoute::new(0, 1023, ARRAY_PREFIX_BITS); + let held_group = ArrayCellRoute::new(0, 15, 16, ARRAY_PREFIX_BITS).group(1); + let applied = ArrayCellRoute::new(0, 1023, 64, ARRAY_PREFIX_BITS); (0..=1023) .filter(|coord| applied.group(*coord) != held_group) .nth(n) diff --git a/nodedb/tests/inproc/cases/calvin_determinism_contract.rs b/nodedb/tests/inproc/cases/calvin_determinism_contract.rs index 01b75b91e..35e743ddb 100644 --- a/nodedb/tests/inproc/cases/calvin_determinism_contract.rs +++ b/nodedb/tests/inproc/cases/calvin_determinism_contract.rs @@ -84,6 +84,7 @@ fn make_request(plan: PhysicalPlan) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/tests/inproc/cases/calvin_executor_apply.rs b/nodedb/tests/inproc/cases/calvin_executor_apply.rs index cab7d9c4d..af5748947 100644 --- a/nodedb/tests/inproc/cases/calvin_executor_apply.rs +++ b/nodedb/tests/inproc/cases/calvin_executor_apply.rs @@ -65,6 +65,7 @@ fn make_request(plan: PhysicalPlan) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply/phantom.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply/phantom.rs index 8f39a4f74..190a659e9 100644 --- a/nodedb/tests/inproc/cases/calvin_two_phase_apply/phantom.rs +++ b/nodedb/tests/inproc/cases/calvin_two_phase_apply/phantom.rs @@ -31,9 +31,8 @@ fn absent_kv_key_phantom_insert_causes_abort() { engine: EngineTag::Kv, collection: "phantomkv".to_string(), key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"newkey".as_slice()))), - read_lsn: Lsn::new(5), + read_version: local_version(5), home_vshard: None, - served_by: 0, }; // Concurrently, the exact same key is inserted and commits at LSN 8. @@ -93,7 +92,7 @@ fn absent_kv_key_phantom_insert_causes_abort() { /// subsequent INSERT of that `document_id` receives, so a per-key OCC check /// would never catch the phantom. The capture layer therefore degrades an /// absent document read to `Predicate` (collection floor). A concurrent INSERT -/// into that collection advances `coll_write_lsn` past the read_lsn, so +/// into that collection advances the collection's version past the read's, so /// `WriteVersionIndex::read_is_valid` (predicate branch) judges the stale read /// invalid and the stage VOTES ABORT — collection-granular phantom safety. #[test] @@ -105,14 +104,13 @@ fn absent_document_phantom_insert_is_caught() { .as_u32(); // The document was absent when read: capture degraded the miss to a - // collection-scoped predicate on "phantomdocs" at read_lsn 5. + // collection-scoped predicate on "phantomdocs" at the version of LSN 5. let absent_doc_read = VersionedReadEntry { engine: EngineTag::Document, collection: "phantomdocs".to_string(), key: ReadKeyIdent::Predicate, - read_lsn: Lsn::new(5), + read_version: local_version(5), home_vshard: None, - served_by: 0, }; // Concurrently, a document with the same document_id the read targeted is @@ -185,9 +183,8 @@ fn absent_document_read_without_matching_insert_still_commits() { engine: EngineTag::Document, collection: "phantomdocs".to_string(), key: ReadKeyIdent::Predicate, - read_lsn: Lsn::new(5), + read_version: local_version(5), home_vshard: None, - served_by: 0, }; // A concurrent insert into a DIFFERENT collection commits at LSN 8. It @@ -212,7 +209,7 @@ fn absent_document_read_without_matching_insert_still_commits() { ); // The predicate read on phantomdocs is still current: its collection floor - // never advanced past read_lsn 5, so the stage must commit. + // never advanced past the read's version, so the stage must commit. let staged = send( &mut core, &mut tx, diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply/read_set.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply/read_set.rs index eef0184a6..bd656cc3f 100644 --- a/nodedb/tests/inproc/cases/calvin_two_phase_apply/read_set.rs +++ b/nodedb/tests/inproc/cases/calvin_two_phase_apply/read_set.rs @@ -16,15 +16,15 @@ use super::support::*; fn drop_discards_invalid_staged_calvin_write() { let (mut core, mut tx, mut rx, _dir) = make_core(); - // Seed a committed write to `dropcoll` at LSN 100 so its collection write - // version floor is 100. The seed carries a WAL LSN so the version records. + // Seed a committed write to `dropcoll` at LSN 100 on its home vShard, so + // its collection version there is the version of LSN 100. let seed = commit_calvin( &mut core, &mut tx, &mut rx, CalvinSeed { epoch: 1, - vshard: 0, + vshard: home_vshard("dropcoll"), collection: "dropcoll", plans: vec![kv_put("dropcoll", b"seed", b"v")], lsn: 100, @@ -34,19 +34,17 @@ fn drop_discards_invalid_staged_calvin_write() { // The read entry's collection must home to the staged request's vShard for // the read-set check to consider it. - let read_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "dropcoll") - .vshard() - .as_u32(); + let read_vshard = home_vshard("dropcoll"); - // A read of `dropcoll` observed at LSN 50 — stale against the seed's write - // at LSN 100 → the read-set is no longer current → abort vote. + // A read of `dropcoll` observed at the version of LSN 50 — stale against + // the seed's write at LSN 100 → the read-set is no longer current → abort + // vote. let stale_read = VersionedReadEntry { engine: EngineTag::Kv, collection: "dropcoll".to_string(), key: ReadKeyIdent::Predicate, - read_lsn: Lsn::new(50), + read_version: local_version(50), home_vshard: None, - served_by: 0, }; let staged = send( @@ -119,10 +117,10 @@ fn drop_discards_invalid_staged_calvin_write() { } /// A `Point` read (not just a `Predicate` read) at or after the recorded write -/// LSN is current → commit vote, and the stamped install applies the staged -/// write. +/// version is current → commit vote, and the stamped install applies the +/// staged write. #[test] -fn point_read_at_write_lsn_commits_and_the_install_applies() { +fn point_read_at_write_version_commits_and_the_install_applies() { let (mut core, mut tx, mut rx, _dir) = make_core(); // Seed a committed write to key `pk` in `pointcoll` at LSN 10. @@ -132,7 +130,7 @@ fn point_read_at_write_lsn_commits_and_the_install_applies() { &mut rx, CalvinSeed { epoch: 1, - vshard: 0, + vshard: home_vshard("pointcoll"), collection: "pointcoll", plans: vec![kv_put("pointcoll", b"pk", b"v1")], lsn: 10, @@ -140,19 +138,16 @@ fn point_read_at_write_lsn_commits_and_the_install_applies() { ); assert_eq!(seed.status, Status::Ok, "seed write must commit: {seed:?}"); - let point_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "pointcoll") - .vshard() - .as_u32(); + let point_vshard = home_vshard("pointcoll"); - // A Point read of the exact same key observed at LSN 10 (== the write) is + // A Point read of the exact same key observed at the write's version is // still current: no write happened AFTER the read. let current_read = VersionedReadEntry { engine: EngineTag::Kv, collection: "pointcoll".to_string(), key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"pk".as_slice()))), - read_lsn: Lsn::new(10), + read_version: local_version(10), home_vshard: None, - served_by: 0, }; let staged = send( @@ -172,7 +167,7 @@ fn point_read_at_write_lsn_commits_and_the_install_applies() { assert_eq!( staged.stage_vote, Some(StageVote::Commit), - "a read at or after the last write LSN must be current -> commit vote" + "a read at or after the last write's version must be current -> commit vote" ); let install_plan = resolved_install(&mut core, &mut tx, &mut rx, 8, point_vshard); @@ -210,7 +205,7 @@ fn point_read_at_write_lsn_commits_and_the_install_applies() { } /// A `Point` read of a key STALE against a later write to that same key -/// (read_lsn=5 vs. a committed write at LSN=10) aborts the stage vote, and the +/// (read at LSN 5 vs. a committed write at LSN 10) aborts the stage vote, and the /// drop discards the staged write with no base mutation — mirrors /// `drop_discards_invalid_staged_calvin_write` but for a Point key (not a /// collection-scoped Predicate). @@ -225,7 +220,7 @@ fn stale_point_read_of_kv_key_aborts_stage_and_drop_discards() { &mut rx, CalvinSeed { epoch: 1, - vshard: 0, + vshard: home_vshard("stalecoll"), collection: "stalecoll", plans: vec![kv_put("stalecoll", b"pk", b"v1")], lsn: 10, @@ -233,9 +228,7 @@ fn stale_point_read_of_kv_key_aborts_stage_and_drop_discards() { ); assert_eq!(seed.status, Status::Ok, "seed write must commit: {seed:?}"); - let stale_vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, "stalecoll") - .vshard() - .as_u32(); + let stale_vshard = home_vshard("stalecoll"); // A Point read of the exact same key observed at LSN 5 — stale against the // write at LSN 10 → the read-set is no longer current → abort vote. @@ -243,9 +236,8 @@ fn stale_point_read_of_kv_key_aborts_stage_and_drop_discards() { engine: EngineTag::Kv, collection: "stalecoll".to_string(), key: ReadKeyIdent::Point(KeyRepr::KvKey(Box::from(b"pk".as_slice()))), - read_lsn: Lsn::new(5), + read_version: local_version(5), home_vshard: None, - served_by: 0, }; let staged = send( @@ -269,7 +261,7 @@ fn stale_point_read_of_kv_key_aborts_stage_and_drop_discards() { assert_eq!( staged.stage_vote, Some(StageVote::SerializationConflict), - "a stale Point read (write after read_lsn) must produce an abort vote" + "a stale Point read (write after the read) must produce an abort vote" ); let before = send( diff --git a/nodedb/tests/inproc/cases/calvin_two_phase_apply/support.rs b/nodedb/tests/inproc/cases/calvin_two_phase_apply/support.rs index 846b6fd45..a2d54c933 100644 --- a/nodedb/tests/inproc/cases/calvin_two_phase_apply/support.rs +++ b/nodedb/tests/inproc/cases/calvin_two_phase_apply/support.rs @@ -17,6 +17,20 @@ use nodedb_types::QualifiedCollection; use nodedb_types::Surrogate; use nodedb_types::calvin::VersionedReadEntry; +/// The version a single-node write at WAL LSN `lsn` records: no data-group +/// entry applied it, so its version is its LSN on top of an empty vShard. +pub(super) fn local_version(lsn: u64) -> nodedb_types::WriteVersion { + nodedb_types::WriteVersion::local_after(nodedb_types::WriteVersion::ZERO, lsn) +} + +/// The vShard a Calvin participant writes `collection` on: the collection's +/// home vShard. +pub(super) fn home_vshard(collection: &str) -> u32 { + nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, collection) + .vshard() + .as_u32() +} + pub(super) fn make_core() -> ( CoreLoop, Producer, @@ -60,6 +74,7 @@ pub(super) fn make_request(plan: PhysicalPlan, vshard: u32, wal_lsn: Option wal_lsn, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/tests/inproc/cases/cross_engine_bitmap_currency.rs b/nodedb/tests/inproc/cases/cross_engine_bitmap_currency.rs index 016b6b2a5..8c034ab21 100644 --- a/nodedb/tests/inproc/cases/cross_engine_bitmap_currency.rs +++ b/nodedb/tests/inproc/cases/cross_engine_bitmap_currency.rs @@ -76,6 +76,7 @@ fn make_req(plan: PhysicalPlan) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/tests/inproc/cases/cross_engine_three_way_fts_vector_doc.rs b/nodedb/tests/inproc/cases/cross_engine_three_way_fts_vector_doc.rs index 1c9391db5..f1718a139 100644 --- a/nodedb/tests/inproc/cases/cross_engine_three_way_fts_vector_doc.rs +++ b/nodedb/tests/inproc/cases/cross_engine_three_way_fts_vector_doc.rs @@ -79,6 +79,7 @@ fn make_req(plan: PhysicalPlan) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/tests/inproc/cases/executor_tests/helpers.rs b/nodedb/tests/inproc/cases/executor_tests/helpers.rs index a725cedda..47cf1d129 100644 --- a/nodedb/tests/inproc/cases/executor_tests/helpers.rs +++ b/nodedb/tests/inproc/cases/executor_tests/helpers.rs @@ -94,6 +94,7 @@ pub fn make_request(plan: PhysicalPlan) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/tests/inproc/cases/request_tracker_backpressure.rs b/nodedb/tests/inproc/cases/request_tracker_backpressure.rs index 011e3a8e5..c481facf4 100644 --- a/nodedb/tests/inproc/cases/request_tracker_backpressure.rs +++ b/nodedb/tests/inproc/cases/request_tracker_backpressure.rs @@ -24,7 +24,7 @@ fn partial(id: u64, data: &[u8]) -> Response { partial: true, payload: Payload::from_vec(data.to_vec()), watermark_lsn: Lsn::ZERO, - read_version_lsn: Lsn::ZERO, + read_versions: nodedb::types::ReadVersions::new(), error_code: None, stage_vote: None, write_set: Vec::new(), diff --git a/nodedb/tests/inproc/cases/startup_gate_pgwire.rs b/nodedb/tests/inproc/cases/startup_gate_pgwire.rs index 71a56fdd1..4c68d6494 100644 --- a/nodedb/tests/inproc/cases/startup_gate_pgwire.rs +++ b/nodedb/tests/inproc/cases/startup_gate_pgwire.rs @@ -84,7 +84,7 @@ fn spawn_fake_data_plane(mut data_side: CoreChannelDataSide) { partial: false, payload, watermark_lsn: Lsn::ZERO, - read_version_lsn: Lsn::ZERO, + read_versions: nodedb::types::ReadVersions::new(), error_code: None, stage_vote: None, write_set: Vec::new(), diff --git a/nodedb/tests/inproc/cases/surrogate_round_trip.rs b/nodedb/tests/inproc/cases/surrogate_round_trip.rs index 841399318..fe0b72849 100644 --- a/nodedb/tests/inproc/cases/surrogate_round_trip.rs +++ b/nodedb/tests/inproc/cases/surrogate_round_trip.rs @@ -90,6 +90,7 @@ fn make_req(plan: PhysicalPlan) -> Request { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Admitted, } } diff --git a/nodedb/tests/inproc/cases/wal_catchup/startup_replay.rs b/nodedb/tests/inproc/cases/wal_catchup/startup_replay.rs index acf1277c7..56ce142ca 100644 --- a/nodedb/tests/inproc/cases/wal_catchup/startup_replay.rs +++ b/nodedb/tests/inproc/cases/wal_catchup/startup_replay.rs @@ -114,6 +114,7 @@ fn startup_replay_recovers_all_wal_data() { wal_lsn: None, resolved_now_ms: None, commit_hlc: None, + entry_version: None, admission: nodedb::bridge::envelope::Admission::Exempt( nodedb::bridge::envelope::ExemptReason::Read, ), From c39e062c375d7583b6fc6bf23099d16336d56d99 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sun, 11 Oct 2026 03:31:53 +0800 Subject: [PATCH 16/19] feat(calvin): run dependent reads through replicated barrier logs Move passive read results of a read-dependent Calvin transaction out of scheduler memory and into a per-vShard barrier log held in the system catalog and replicated through the data group. Every replica of a vShard folds the same barrier entries, so a follower promoted to leader, or a node restarted mid-barrier, resumes the transaction from the log rather than losing the reads. A barrier that waits past dependent_read_passive_timeout_ms is resolved by a timeout entry the data-group leader proposes, so every replica aborts it alike. The default timeout rises to 5000 ms, and max_inflight_backlog also bounds the barrier entries held in memory for transactions the scheduler has not taken yet. The planner builds read-dependent and predicted transaction classes separately. TRANSFER_ITEM between collections on two vShards now runs as one read-dependent Calvin transaction instead of returning an error. --- docs/getting-started.md | 10 +- .../cluster_suite/cases/calvin_e2e_ollp.rs | 6 +- .../cases/calvin_dependent_read_commit.rs | 56 ++ .../calvin_dependent_read_early_result.rs | 124 +++ .../cases/calvin_dependent_read_fixture.rs | 285 +++++++ ...alvin_dependent_read_follower_promotion.rs | 147 ++++ .../cases/calvin_dependent_read_restart.rs | 192 +++++ .../cases/calvin_dependent_read_timeout.rs | 105 +++ .../tests/common_suite/cases/mod.rs | 6 + nodedb-cluster/src/calvin/sequencer/inbox.rs | 2 + nodedb-cluster/src/calvin/types/mod.rs | 4 +- nodedb-cluster/src/calvin/types/primitives.rs | 67 +- .../src/calvin/types/transaction.rs | 20 +- nodedb-cluster/src/raft_loop/proposals.rs | 12 +- nodedb-cluster/src/rpc_codec/data_propose.rs | 30 + nodedb-cluster/src/rpc_codec/mod.rs | 4 +- nodedb-physical/src/physical_plan/meta.rs | 19 +- .../src/physical_plan/meta_calvin.rs | 35 +- nodedb-types/src/calvin.rs | 13 +- nodedb-types/src/calvin_passive.rs | 110 +++ nodedb-types/src/config/tuning/calvin.rs | 23 +- nodedb-types/src/lib.rs | 1 + .../cluster/array_cluster_exec/dispatch.rs | 8 +- nodedb/src/control/cluster/calvin/mod.rs | 2 +- .../cluster/calvin/scheduler/barrier_log.rs | 279 ++++++ .../cluster/calvin/scheduler/barrier_store.rs | 387 +++++++++ .../calvin/scheduler/driver/barrier.rs | 170 ++-- .../cluster/calvin/scheduler/driver/config.rs | 11 +- .../driver/core/commit_resolve/verdict.rs | 11 + .../scheduler/driver/core/completion_route.rs | 3 + .../calvin/scheduler/driver/core/deferred.rs | 11 +- .../driver/core/dispatch/active_dispatch.rs | 11 +- .../scheduler/driver/core/dispatch/follow.rs | 24 +- .../calvin/scheduler/driver/core/halt.rs | 4 + .../calvin/scheduler/driver/core/mod.rs | 9 +- .../scheduler/driver/core/passive_read.rs | 491 +++++++++++ .../calvin/scheduler/driver/core/process.rs | 56 +- .../calvin/scheduler/driver/core/propose.rs | 109 ++- .../scheduler/driver/core/read_result.rs | 804 +++++++++++++----- .../scheduler/driver/core/read_timeout.rs | 122 +++ .../scheduler/driver/core/redo_applied.rs | 7 +- .../scheduler/driver/core/redo_propose.rs | 4 +- .../calvin/scheduler/driver/core/restage.rs | 4 +- .../calvin/scheduler/driver/core/role.rs | 37 +- .../calvin/scheduler/driver/core/run_loop.rs | 40 +- .../calvin/scheduler/driver/core/scheduler.rs | 31 +- .../scheduler/driver/core/test_support.rs | 9 - .../cluster/calvin/scheduler/driver/mod.rs | 1 - .../cluster/calvin/scheduler/driver/types.rs | 5 + .../scheduler/lock/manager/introspection.rs | 57 +- .../calvin/scheduler/lock/manager/mod.rs | 4 +- .../cluster/calvin/scheduler/lock/mod.rs | 2 +- .../control/cluster/calvin/scheduler/mod.rs | 9 +- .../cluster/calvin/scheduler/read_results.rs | 511 +++++++++++ .../cluster/calvin_snapshot/capture.rs | 26 +- .../cluster/calvin_snapshot/install.rs | 8 + .../src/control/cluster/snapshot_builder.rs | 3 +- nodedb/src/control/cluster/start_raft/core.rs | 1 - .../control/cluster/start_raft/group_setup.rs | 9 +- .../src/control/cluster/start_raft/hooks.rs | 7 + .../control/cluster/start_raft/loop_build.rs | 9 +- .../cluster/start_raft/propose_error.rs | 15 +- .../cluster/start_raft/proposer_wiring.rs | 17 +- .../src/control/cluster/start_raft_helpers.rs | 72 +- .../apply_loop/barrier_prefold.rs | 40 + .../apply_loop/calvin_read_result.rs | 507 +++++++++-- .../distributed_applier/apply_loop/context.rs | 10 +- .../distributed_applier/apply_loop/driver.rs | 5 - .../distributed_applier/apply_loop/lane.rs | 10 +- .../apply_loop/metadata_floor.rs | 7 + .../distributed_applier/apply_loop/mod.rs | 8 +- .../apply_loop/pipeline.rs | 7 + .../apply_loop/redo_chunk.rs | 7 +- .../distributed_applier/apply_loop/start.rs | 32 +- nodedb/src/control/gateway/cluster_error.rs | 12 +- nodedb/src/control/gateway/dispatch_remote.rs | 33 +- .../control/planner/calvin/dependent_recon.rs | 6 +- nodedb/src/control/planner/calvin/mod.rs | 5 +- .../calvin/read_dependent/item_move.rs | 247 ++++++ .../planner/calvin/read_dependent/mod.rs | 9 + .../planner/calvin/read_dependent/recon.rs | 47 + .../control/planner/calvin/submit/assign.rs | 8 +- .../control/planner/calvin/submit/routed.rs | 13 +- .../control/planner/calvin/tx_class/mod.rs | 15 +- ...endent_builder.rs => predicted_builder.rs} | 51 +- .../calvin/tx_class/read_dependent_builder.rs | 230 +++++ .../calvin/tx_class/write_keys/crdt.rs | 56 +- .../security/catalog/bootstrap_tables.rs | 1 + .../security/catalog/calvin_barrier_logs.rs | 300 +++++++ nodedb/src/control/security/catalog/mod.rs | 1 + .../security/catalog/replicated_image.rs | 2 + .../shared/ddl/neutral/kv_atomic/dispatch.rs | 2 +- .../shared/ddl/neutral/kv_atomic/mod.rs | 2 +- .../control/server/shared/ddl/neutral/mod.rs | 1 + .../server/shared/ddl/neutral/transfer.rs | 29 +- .../ddl/neutral/transfer_cross_shard.rs | 130 +++ .../server/shared/write_admission/gate.rs | 41 +- .../shared/write_admission/leader_gate.rs | 1 + .../server/shared/write_admission/wait.rs | 9 + nodedb/src/control/state/calvin_local.rs | 7 +- nodedb/src/control/state/methods.rs | 22 +- nodedb/src/control/state/proposer_pair.rs | 4 +- nodedb/src/control/vshard_admission.rs | 16 +- .../control/wal_replication/decode/entry.rs | 4 + .../wal_replication/types/replicated_write.rs | 10 + .../handlers/control/calvin/active_passive.rs | 166 +++- .../executor/handlers/control/calvin/mod.rs | 8 +- .../handlers/control/calvin_passive_read.rs | 204 ++--- nodedb/src/diag/context/mod.rs | 2 +- nodedb/src/diag/context/raft_apply.rs | 53 ++ nodedb/src/diag/mod.rs | 33 +- nodedb/src/diag/recording/mod.rs | 4 +- nodedb/src/diag/recording/raft_apply.rs | 28 +- nodedb/src/error_from.rs | 12 +- nodedb/src/types/mod.rs | 2 +- nodedb/src/types/snapshot_calvin.rs | 25 + .../inproc/cases/calvin_executor_caps.rs | 4 + .../cases/calvin_executor_dependent_read.rs | 293 ++----- 118 files changed, 6297 insertions(+), 1155 deletions(-) create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_commit.rs create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_early_result.rs create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_fixture.rs create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_follower_promotion.rs create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_restart.rs create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_timeout.rs create mode 100644 nodedb-types/src/calvin_passive.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/barrier_log.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/barrier_store.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/passive_read.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/driver/core/read_timeout.rs create mode 100644 nodedb/src/control/cluster/calvin/scheduler/read_results.rs create mode 100644 nodedb/src/control/distributed_applier/apply_loop/barrier_prefold.rs create mode 100644 nodedb/src/control/planner/calvin/read_dependent/item_move.rs create mode 100644 nodedb/src/control/planner/calvin/read_dependent/mod.rs create mode 100644 nodedb/src/control/planner/calvin/read_dependent/recon.rs rename nodedb/src/control/planner/calvin/tx_class/{dependent_builder.rs => predicted_builder.rs} (88%) create mode 100644 nodedb/src/control/planner/calvin/tx_class/read_dependent_builder.rs create mode 100644 nodedb/src/control/security/catalog/calvin_barrier_logs.rs create mode 100644 nodedb/src/control/server/shared/ddl/neutral/transfer_cross_shard.rs diff --git a/docs/getting-started.md b/docs/getting-started.md index b33cd2215..c8ffa0233 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -595,7 +595,7 @@ Below that the sweep costs more than the resolution it buys. | ------------------------------------------------ | ------- | | `tuning.calvin.channel_capacity` | `512` | | `tuning.calvin.txn_deadline_multiplier` | `3` | -| `tuning.calvin.dependent_read_passive_timeout_ms` | `60` | +| `tuning.calvin.dependent_read_passive_timeout_ms` | `5000` | | `tuning.calvin.verdict_stall_warn_ms` | `5000` | | `tuning.calvin.max_inflight_backlog` | `1024` | | `tuning.calvin.catch_up_window` | `512` | @@ -607,11 +607,15 @@ Each hosted vShard runs one Calvin scheduler with these settings. - `channel_capacity` sizes each scheduler's bounded input and completion channels. - `txn_deadline_multiplier` sets a transaction deadline in sequencer epochs. -- `dependent_read_passive_timeout_ms` bounds the wait for passive read results. +- `dependent_read_passive_timeout_ms` bounds the wait of a dependent-read + barrier for passive read results. Past it, the vShard's data-group leader + proposes a timeout entry, and every replica aborts the barrier alike. - `verdict_stall_warn_ms` spaces the stall warnings of a transaction waiting on its global verdict. The transaction keeps waiting and never aborts. - `max_inflight_backlog` is the in-flight backlog at which a scheduler stops - taking new input. + taking new input. It also bounds the dependent-read barrier entries a + vShard holds in memory for transactions its scheduler has not taken yet. + Entries past it wait in the system catalog alone. - `catch_up_window` is the most sequencer log entries one catch-up pass replays. - `restage_attempts` is the most times a data-group leader stages a diff --git a/nodedb-cluster-tests/tests/cluster_suite/cases/calvin_e2e_ollp.rs b/nodedb-cluster-tests/tests/cluster_suite/cases/calvin_e2e_ollp.rs index f71e163d5..216ba73c8 100644 --- a/nodedb-cluster-tests/tests/cluster_suite/cases/calvin_e2e_ollp.rs +++ b/nodedb-cluster-tests/tests/cluster_suite/cases/calvin_e2e_ollp.rs @@ -13,7 +13,7 @@ //! surrogates currently matching the predicate ("predicted surrogates"). //! 2. The TxClass is built with the predicted surrogates for the OLLP //! collection and static surrogates for any additional static-key writes -//! in the same transaction (via `build_dependent_tx_class`), then submitted +//! in the same transaction (via `build_predicted_tx_class`), then submitted //! to the sequencer inbox. //! 3. The sequencer admits the txn and fans it out to all participant vshards. //! 4. The Data Plane executor verifies the predicted set against the actual @@ -71,7 +71,7 @@ fn two_distinct_vshard_collections() -> (String, String) { /// - An OLLP BulkUpdate on `col_ollp` with `predicted_surrogates` as the /// write set (since the exact set is not known statically). /// -/// This mirrors what `build_dependent_tx_class` produces on the Control Plane +/// This mirrors what `build_predicted_tx_class` produces on the Control Plane /// for a cross-shard transaction that contains a value-dependent predicate write. fn make_ollp_tx_class( col_static: &str, @@ -94,7 +94,7 @@ fn make_ollp_tx_class( write_set, // Empty plan bytes — this test exercises the sequencer admission and // fan-out path only. The actual BulkUpdate plan bytes are embedded by - // `build_dependent_tx_class` in production. + // `build_predicted_tx_class` in production. vec![], TenantId::new(1), None, diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_commit.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_commit.rs new file mode 100644 index 000000000..58dd8c80d --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_commit.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A cross-shard write whose value depends on a row of another vShard +//! commits on every replica as one dependent-read Calvin transaction. +//! +//! `SELECT TRANSFER_ITEM(src, dst, 'sword', 'alice', 'bob')` moves an item +//! between key-value collections on two vShards. The source vShard reads the +//! item under the transaction's locks and broadcasts it through the +//! data-group log of both active vShards. Every replica of the source then +//! lacks the item, and every replica of the destination holds the bytes the +//! source held. A second move of the same item finds it gone. + +use super::calvin_dependent_read_fixture::{ItemMove, assert_no_apply_halt}; +use super::calvin_replica_content::strict_session; +use crate::common::cluster_harness::shared_steps::db_detail; + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_cross_shard_item_move_commits_on_every_replica() { + let mv = ItemMove::spawn("dep_commit").await; + let source_bytes = mv.source_bytes().await; + let session = strict_session(mv.fx.coordinator()).await; + + let reply = session + .simple_query(&mv.transfer_sql()) + .await + .unwrap_or_else(|e| panic!("the move commits: {}", db_detail(&e))); + let text: Vec = reply + .into_iter() + .filter_map(|msg| match msg { + tokio_postgres::SimpleQueryMessage::Row(row) => row.get(0).map(str::to_owned), + _ => None, + }) + .collect(); + assert_eq!(text.len(), 1, "the move answers one row"); + assert!( + text[0].contains("bob:sword") && text[0].contains(&mv.dest), + "the answer names the item's new home: {}", + text[0] + ); + + mv.wait_replicas_agree(Some(&source_bytes)).await; + + let again = session + .simple_query(&mv.transfer_sql()) + .await + .expect_err("the item left the source"); + let detail = db_detail(&again).to_lowercase(); + assert!( + detail.contains("not found") || detail.contains("not_found") || detail.contains("22023"), + "a second move reports the item missing, got: {detail}" + ); + + assert_no_apply_halt(mv.cluster()); + let ItemMove { fx, .. } = mv; + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_early_result.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_early_result.rs new file mode 100644 index 000000000..ecdb7738e --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_early_result.rs @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A read result that reaches a vShard before the vShard granted its txn is +//! kept, and the barrier the grant opens completes from it. +//! +//! 1. A transaction `P` writes `bob:sword` at the destination and a row at +//! the source. The fail gate `calvin::before_redo_propose::` holds +//! its destination redo, so every destination replica keeps `P`'s lock +//! on `bob:sword`. +//! 2. The item move `T` queues behind that lock on the destination. Its +//! source vShard holds no conflicting lock, reads the item, and +//! broadcasts it. The broadcast applies on every destination replica +//! before the replica granted `T`. It waits in the replica's read-result +//! buffer and in its stored barrier row. +//! 3. Released, `P` commits and frees the lock. Each replica grants `T` and +//! takes the buffered result. The destination's leader stages at once, +//! and `T` commits with the item's bytes on every replica. Each replica +//! then drops the txn's buffered entries and its row. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::time::Duration; + +use nodedb_types::fail_point::{FailAction, FailGuard}; + +use super::calvin_dependent_read_fixture::{ + ItemMove, assert_no_apply_halt, stored_barrier_rows, waiting_barrier_txns, +}; +use super::calvin_replica_content::strict_session; +use crate::common::cluster_harness::shared_steps::db_detail; +use crate::common::cluster_harness::wait_for; + +/// How long the test waits for a held step. +const HOLD_DEADLINE: Duration = Duration::from_secs(30); + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_read_result_before_the_grant_is_applied_not_dropped() { + let mv = ItemMove::spawn("dep_early").await; + let source_bytes = mv.source_bytes().await; + let dest_group = mv.group_of(&mv.dest); + let dest_vshard = mv.dest_vshard(); + let holder = strict_session(mv.fx.coordinator()).await; + let mover = strict_session(mv.fx.coordinator()).await; + + let gate_dir = tempfile::tempdir().expect("gate directory"); + let release = gate_dir.path().join("release"); + let parked = gate_dir.path().join("release.parked"); + let gate = FailGuard::install( + &format!("calvin::before_redo_propose::{}", mv.dest), + FailAction::WaitForFile(release.clone()), + ); + + let hold_lock = format!( + "BEGIN; \ + INSERT INTO {dest} (key, owner, note) VALUES ('bob:sword', 'bob', 'old'); \ + INSERT INTO {source} (key, owner, note) VALUES ('carol:shield', 'carol', 'x'); \ + COMMIT", + dest = mv.dest, + source = mv.source, + ); + let transfer = mv.transfer_sql(); + let (held, moved, ()) = tokio::join!( + holder.simple_query(&hold_lock), + async { + wait_for( + "the destination leader holds P's redo, and with it the lock", + HOLD_DEADLINE, + Duration::from_millis(20), + || parked.exists(), + ) + .await; + mover.simple_query(&transfer).await + }, + async { + wait_for( + "every destination replica buffers the read result before its grant", + HOLD_DEADLINE, + Duration::from_millis(20), + || { + parked.exists() + && mv + .cluster() + .nodes + .iter() + .filter(|node| node.replicates_data_group(dest_group)) + .all(|node| { + waiting_barrier_txns(node, dest_vshard) >= 1 + && stored_barrier_rows(node, dest_vshard) >= 1 + }) + }, + ) + .await; + std::fs::write(&release, b"release").expect("release P's redo"); + }, + ); + held.unwrap_or_else(|e| panic!("P commits: {}", db_detail(&e))); + moved.unwrap_or_else(|e| { + panic!( + "the move commits from the buffered result: {}", + db_detail(&e) + ) + }); + drop(gate); + + mv.wait_replicas_agree(Some(&source_bytes)).await; + wait_for( + "every node drops the barrier entries and rows of the finished move", + HOLD_DEADLINE, + Duration::from_millis(20), + || { + mv.cluster().nodes.iter().all(|node| { + waiting_barrier_txns(node, dest_vshard) == 0 + && stored_barrier_rows(node, dest_vshard) == 0 + }) + }, + ) + .await; + assert_no_apply_halt(mv.cluster()); + + let ItemMove { fx, .. } = mv; + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_fixture.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_fixture.rs new file mode 100644 index 000000000..623d4a558 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_fixture.rs @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Shared fixture for the Calvin dependent-read barrier cases. +//! +//! `SELECT TRANSFER_ITEM(src, dst, 'sword', 'alice', 'bob')` between two +//! key-value collections on distinct vShards is a read-dependent Calvin +//! transaction. The source vShard reads `alice:sword` under the +//! transaction's locks and broadcasts it, through the data-group log of each +//! active vShard, to the delete at the source and the put at the +//! destination. +//! +//! Each replica's content is read from its own Data-Plane cores, so a check +//! sees what that replica installed, never a row a gateway fetched. + +use std::collections::BTreeMap; +use std::time::Duration; + +use nodedb::types::TenantId; + +use super::calvin_multishard_fixture::Fixture; +use super::calvin_replica_content::names_on_distinct_vshards; +#[cfg(feature = "failpoints")] +use super::calvin_replica_content::vshard_of; +use crate::common::cluster_harness::shared_steps::{db_detail, leader_of}; +use crate::common::cluster_harness::{TestCluster, TestClusterNode, wait_for_async}; + +/// The tenant the `nodedb` pgwire user writes as. +const TENANT: u64 = 1; + +/// The item's key at the source. +pub(super) const SOURCE_KEY: &[u8] = b"alice:sword"; + +/// The item's key at the destination. +pub(super) const DEST_KEY: &[u8] = b"bob:sword"; + +/// How long the replicas get to converge on the move's outcome. +const CONVERGE: Duration = Duration::from_secs(30); + +/// A cluster with a source and a destination key-value collection on +/// distinct vShards, the item stored at the source. +pub(super) struct ItemMove { + pub(super) fx: Fixture, + pub(super) source: String, + pub(super) dest: String, +} + +impl ItemMove { + /// Spawn the cluster, create both collections, and store the item. + pub(super) async fn spawn(prefix: &str) -> Self { + let source_prefix = format!("{prefix}_src"); + let dest_prefix = format!("{prefix}_dst"); + let [source, dest] = + names_on_distinct_vshards([source_prefix.as_str(), dest_prefix.as_str()]); + let fx = Fixture::spawn(&[kv_ddl(&source), kv_ddl(&dest)]).await; + fx.wait_group_mounted(&source).await; + fx.wait_group_mounted(&dest).await; + // Collection creation installs each collection's constraint set on + // every replica in the background. The install is one more write of + // the collection, so the cases start their txns once it applied. + wait_constraints_installed(&fx.cluster, &source).await; + wait_constraints_installed(&fx.cluster, &dest).await; + fx.coordinator() + .client + .simple_query(&format!( + "INSERT INTO {source} (key, owner, note) VALUES ('alice:sword', 'alice', 'forged')" + )) + .await + .unwrap_or_else(|e| panic!("store the item: {}", db_detail(&e))); + fx.converge().await; + Self { fx, source, dest } + } + + /// The statement that moves the item from the source to the + /// destination. + pub(super) fn transfer_sql(&self) -> String { + format!( + "SELECT TRANSFER_ITEM('{}', '{}', 'sword', 'alice', 'bob')", + self.source, self.dest + ) + } + + pub(super) fn cluster(&self) -> &TestCluster { + &self.fx.cluster + } + + /// The vShard of the source collection. + #[cfg(feature = "failpoints")] + pub(super) fn source_vshard(&self) -> u32 { + vshard_of(&self.source) + } + + /// The vShard of the destination collection. + #[cfg(feature = "failpoints")] + pub(super) fn dest_vshard(&self) -> u32 { + vshard_of(&self.dest) + } + + /// The data group of `collection`, as the first node routes it. + pub(super) fn group_of(&self, collection: &str) -> u64 { + self.cluster().nodes[0] + .group_id_for_collection(collection) + .unwrap_or_else(|| panic!("no data group for {collection}")) + } + + /// Wait until every replica agrees on the move's outcome. With + /// `moved = None`, every source replica holds the item and no + /// destination replica holds it. With `moved = Some(bytes)`, no source + /// replica holds the item and every destination replica holds `bytes` + /// for it: the bytes the source held, not merely some row at the key. + pub(super) async fn wait_replicas_agree(&self, moved: Option<&[u8]>) { + let source_group = self.group_of(&self.source); + let dest_group = self.group_of(&self.dest); + let deadline = std::time::Instant::now() + CONVERGE; + loop { + let mut agree = true; + let mut views = Vec::new(); + for node in &self.cluster().nodes { + let source_holds = node.replicates_data_group(source_group) + && local_kv_rows(node, &self.source) + .await + .contains_key(SOURCE_KEY); + let dest_bytes = if node.replicates_data_group(dest_group) { + local_kv_rows(node, &self.dest).await.remove(DEST_KEY) + } else { + moved.map(<[u8]>::to_vec) + }; + if (node.replicates_data_group(source_group) && source_holds == moved.is_some()) + || dest_bytes.as_deref() != moved + { + agree = false; + } + views.push(format!( + "node {}: source holds the item = {source_holds}, destination bytes match = \ + {}, source group applied {} (leader {}), destination group applied {} \ + (leader {})", + node.node_id, + dest_bytes.as_deref() == moved, + node.shared.applied_index_watcher(source_group).current(), + leader_of(node, source_group), + node.shared.applied_index_watcher(dest_group).current(), + leader_of(node, dest_group), + )); + } + if agree { + return; + } + assert!( + std::time::Instant::now() < deadline, + "the replicas disagree on the move (moved = {}) after {CONVERGE:?}:\n{}", + moved.is_some(), + views.join("\n") + ); + tokio::time::sleep(Duration::from_millis(200)).await; + } + } + + /// The bytes the source's replicas hold for the item. Panics when two + /// replicas disagree or none holds it. + pub(super) async fn source_bytes(&self) -> Vec { + let source_group = self.group_of(&self.source); + let mut seen: Option> = None; + for node in &self.cluster().nodes { + if !node.replicates_data_group(source_group) { + continue; + } + let bytes = local_kv_rows(node, &self.source) + .await + .remove(SOURCE_KEY) + .unwrap_or_else(|| panic!("node {} does not hold the item", node.node_id)); + if let Some(first) = &seen { + assert_eq!(first, &bytes, "the source's replicas disagree"); + } + seen = Some(bytes); + } + seen.expect("the source group has a replica") + } +} + +/// Assert no node halted a Calvin scheduler. +pub(super) fn assert_no_apply_halt(cluster: &TestCluster) { + for node in &cluster.nodes { + let halt = node.shared.sequencer_halt.apply_halt().report(); + assert!( + halt.is_none(), + "node {} halted a Calvin scheduler: {halt:?}", + node.node_id + ); + } +} + +/// The txns of `vshard` whose barrier entries `node` holds and no scheduler +/// took, in memory or in their stored row. +#[cfg(feature = "failpoints")] +pub(super) fn waiting_barrier_txns(node: &TestClusterNode, vshard: u32) -> usize { + node.shared.calvin.read_results.waiting_txns(vshard) +} + +/// The barrier rows `node` stores for `vshard`: one per txn whose barrier +/// entry applied there and did not finish. +#[cfg(feature = "failpoints")] +pub(super) fn stored_barrier_rows(node: &TestClusterNode, vshard: u32) -> usize { + node.shared + .credentials + .catalog() + .load_calvin_barrier_logs(Some(&std::collections::BTreeSet::from([vshard]))) + .expect("the barrier rows load") + .len() +} + +/// The `(group, log index)` of the latest read-result entries `node` +/// applied for `vshard` in its current life. +#[cfg(feature = "failpoints")] +pub(super) fn read_entries(node: &TestClusterNode, vshard: u32) -> Vec<(u64, u64)> { + node.shared + .calvin + .read_results + .vshard(vshard) + .read_entries() +} + +/// The read-result entries `node` applied for `vshard`. +#[cfg(feature = "failpoints")] +pub(super) fn reads_applied(node: &TestClusterNode, vshard: u32) -> u64 { + node.shared + .calvin + .read_results + .vshard(vshard) + .reads_applied() +} + +/// Wait until every replica of `collection`'s data group installed the +/// collection's constraint set in its CRDT validator. The set of a key-value +/// collection names its key `NOT NULL`, so it is never empty. +async fn wait_constraints_installed(cluster: &TestCluster, collection: &str) { + let group = cluster.nodes[0] + .group_id_for_collection(collection) + .unwrap_or_else(|| panic!("no data group for {collection}")); + wait_for_async( + &format!("every replica of {collection} installed its constraint set"), + CONVERGE, + Duration::from_millis(100), + || async move { + for node in &cluster.nodes { + if node.replicates_data_group(group) + && node + .crdt_constraints(TenantId::new(TENANT), collection) + .await + .is_empty() + { + return false; + } + } + true + }, + ) + .await; +} + +fn kv_ddl(collection: &str) -> String { + format!( + "CREATE COLLECTION {collection} (key TEXT PRIMARY KEY, owner TEXT, note TEXT) \ + WITH (engine='kv')" + ) +} + +/// Every key-value row of `collection` as `node` stores it: key to stored +/// bytes, read from each of its Data-Plane cores. +async fn local_kv_rows(node: &TestClusterNode, collection: &str) -> BTreeMap, Vec> { + let table = format!("0:{TENANT}:{collection}"); + let mut rows = BTreeMap::new(); + for core in 0..node.num_cores() { + let snapshot = node + .tenant_snapshot_on_core(core, TenantId::new(TENANT)) + .await; + for (key, bytes) in snapshot.kv_tables { + if key != table { + continue; + } + let entries: Vec<(Vec, Vec, u64, u32)> = + zerompk::from_msgpack(&bytes).expect("KV table snapshot decodes"); + rows.extend(entries.into_iter().map(|(key, value, _, _)| (key, value))); + } + } + rows +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_follower_promotion.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_follower_promotion.rs new file mode 100644 index 000000000..97210f163 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_follower_promotion.rs @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A follower that buffered a dependent-read result before its grant does +//! not halt once it leads: the barrier decides from the log, not a clock. +//! +//! 1. Node `F` leads neither the source's nor the destination's data group. +//! Its metadata apply is held, and a DDL through another node moves the +//! metadata log past it. The item move `T` carries the coordinator's +//! metadata floor, so `F`'s schedulers hold `T` ungranted. +//! 2. The source vShard reads the item and broadcasts it. The broadcast +//! applies on `F` before `F` granted `T`. The destination's leader stages +//! `T` and votes commit, and the fail gate +//! `calvin::before_redo_propose::` holds its redo. +//! 3. `F` is released and grants `T` as a follower, and takes the buffered +//! result. Leadership of the destination's group moves to `F`. `F` opens +//! the barrier from the result, stages `T`, and proposes its redo once +//! the gate is released. +//! 4. `T` commits on every replica, and no node halts a scheduler with +//! `LocalStageFailed`. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::time::Duration; + +use nodedb::control::cluster::metadata_applier::metadata_apply_hold_point; +use nodedb_types::fail_point::{FailAction, FailGuard}; + +use super::calvin_dependent_read_fixture::{ItemMove, assert_no_apply_halt, waiting_barrier_txns}; +use super::calvin_replica_content::strict_session; +use crate::common::cluster_harness::shared_steps::{db_detail, leader_of}; +use crate::common::cluster_harness::wait_for; + +/// How long the test waits for a held step. +const HOLD_DEADLINE: Duration = Duration::from_secs(30); + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_promoted_follower_commits_the_move_without_halting() { + let mv = ItemMove::spawn("dep_promote").await; + mv.cluster().wait_for_preferred_leaders(HOLD_DEADLINE).await; + let source_bytes = mv.source_bytes().await; + let source_group = mv.group_of(&mv.source); + let dest_group = mv.group_of(&mv.dest); + let dest_vshard = mv.dest_vshard(); + let nodes = &mv.cluster().nodes; + let source_leader = leader_of(&nodes[0], source_group); + let dest_leader = leader_of(&nodes[0], dest_group); + let follower = nodes + .iter() + .position(|node| node.node_id != source_leader && node.node_id != dest_leader) + .expect("three nodes hold a node that leads neither group"); + let follower_id = nodes[follower].node_id; + let coordinator = (0..nodes.len()) + .find(|idx| *idx != follower) + .expect("a coordinator besides the follower"); + let session = strict_session(&nodes[coordinator]).await; + + // Hold the follower's metadata apply and move the metadata log past it. + let meta_dir = tempfile::tempdir().expect("metadata gate directory"); + let meta_release = meta_dir.path().join("release"); + let meta_parked = meta_dir.path().join("release.parked"); + let _meta_hold = FailGuard::install( + &metadata_apply_hold_point(follower_id), + FailAction::WaitForFile(meta_release.clone()), + ); + nodes[coordinator] + .exec("CREATE TOPIC dep_promote_lag") + .await + .unwrap_or_else(|e| panic!("the DDL commits without the follower: {e}")); + let metadata = nodedb_cluster::METADATA_GROUP_ID; + wait_for( + "the follower's metadata apply lags the coordinator's", + HOLD_DEADLINE, + Duration::from_millis(20), + || { + meta_parked.exists() + && nodes[follower] + .shared + .applied_index_watcher(metadata) + .current() + < nodes[coordinator] + .shared + .applied_index_watcher(metadata) + .current() + }, + ) + .await; + + let redo_dir = tempfile::tempdir().expect("redo gate directory"); + let redo_release = redo_dir.path().join("release"); + let redo_parked = redo_dir.path().join("release.parked"); + let redo_gate = FailGuard::install( + &format!("calvin::before_redo_propose::{}", mv.dest), + FailAction::WaitForFile(redo_release.clone()), + ); + + let transfer = mv.transfer_sql(); + let (moved, ()) = tokio::join!(session.simple_query(&transfer), async { + // Two waits, so a timeout names the half that never held. + wait_for( + "the follower buffered the read result before its grant", + HOLD_DEADLINE, + Duration::from_millis(20), + || waiting_barrier_txns(&nodes[follower], dest_vshard) >= 1, + ) + .await; + wait_for( + "the destination leader voted commit and holds the redo", + HOLD_DEADLINE, + Duration::from_millis(20), + || redo_parked.exists(), + ) + .await; + assert!( + nodes[follower] + .shared + .applied_index_watcher(metadata) + .current() + < nodes[coordinator] + .shared + .applied_index_watcher(metadata) + .current(), + "the follower's metadata apply still lags, so its grant waits" + ); + std::fs::write(&meta_release, b"release").expect("release the follower's metadata"); + wait_for( + "the follower granted the move and took its buffered result", + HOLD_DEADLINE, + Duration::from_millis(20), + || waiting_barrier_txns(&nodes[follower], dest_vshard) == 0, + ) + .await; + mv.cluster() + .transfer_leadership(dest_group, follower_id) + .await; + std::fs::write(&redo_release, b"release").expect("release the redo"); + }); + moved.unwrap_or_else(|e| panic!("the move commits under the new leader: {}", db_detail(&e))); + drop(redo_gate); + + mv.wait_replicas_agree(Some(&source_bytes)).await; + assert_no_apply_halt(mv.cluster()); + + let ItemMove { fx, .. } = mv; + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_restart.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_restart.rs new file mode 100644 index 000000000..a6f5f6e37 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_restart.rs @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A replica that restarts after a dependent-read result applied, and before +//! the txn finished, reaches the barrier outcome its peers reached: the +//! result's stored row survives the restart. +//! +//! 1. Node `R` leads neither the source's nor the destination's data group. +//! The fail gate `calvin::before_redo_propose::` holds the +//! destination redo of the item move `T`, so `T` does not finish on the +//! destination. +//! 2. The source vShard's read result applies on `R` and lands in its stored +//! barrier row. The test pins the result's entry by its log index. A +//! later write to the destination group moves `R`'s durable applied +//! floor past that index, so a restart never delivers the entry again. +//! 3. `R` stops and starts again, and applies no pinned entry again. A +//! duplicate entry at a later index can apply. Leadership of the +//! destination's group moves to `R`. `R` opens `T`'s barrier from the +//! stored row, stages `T`, and proposes its redo once the gate is +//! released. +//! 4. `T` commits on every replica, and no node halts a scheduler. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::time::Duration; + +use nodedb_types::fail_point::{FailAction, FailGuard}; + +use super::calvin_dependent_read_fixture::{ + ItemMove, assert_no_apply_halt, read_entries, stored_barrier_rows, waiting_barrier_txns, +}; +use super::calvin_replica_content::strict_session; +use crate::common::cluster_harness::shared_steps::{db_detail, leader_of}; +use crate::common::cluster_harness::wait_for; + +/// How long the test waits for a held step. +const HOLD_DEADLINE: Duration = Duration::from_secs(30); + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_restarted_replica_reaches_the_barrier_outcome_from_its_stored_row() { + let mut mv = ItemMove::spawn("dep_restart").await; + mv.cluster().wait_for_preferred_leaders(HOLD_DEADLINE).await; + let source_bytes = mv.source_bytes().await; + let source_group = mv.group_of(&mv.source); + let dest_group = mv.group_of(&mv.dest); + let dest_vshard = mv.dest_vshard(); + let (restarted, restarted_id, coordinator) = { + let nodes = &mv.cluster().nodes; + let source_leader = leader_of(&nodes[0], source_group); + let dest_leader = leader_of(&nodes[0], dest_group); + let restarted = nodes + .iter() + .position(|node| { + node.node_id != source_leader + && node.node_id != dest_leader + && node.replicates_data_group(dest_group) + }) + .expect("three nodes hold a destination replica that leads neither group"); + let coordinator = (0..nodes.len()) + .find(|idx| *idx != restarted) + .expect("a coordinator besides the restarted node"); + (restarted, nodes[restarted].node_id, coordinator) + }; + let session = strict_session(&mv.cluster().nodes[coordinator]).await; + let filler = strict_session(&mv.cluster().nodes[coordinator]).await; + + let redo_dir = tempfile::tempdir().expect("redo gate directory"); + let redo_release = redo_dir.path().join("release"); + let redo_parked = redo_dir.path().join("release.parked"); + let redo_gate = FailGuard::install( + &format!("calvin::before_redo_propose::{}", mv.dest), + FailAction::WaitForFile(redo_release.clone()), + ); + + let transfer = mv.transfer_sql(); + let filler_sql = format!( + "INSERT INTO {} (key, owner, note) VALUES ('dave:axe', 'dave', 'filler')", + mv.dest + ); + let (moved, ()) = tokio::join!(session.simple_query(&transfer), async { + { + let node = &mv.cluster().nodes[restarted]; + wait_for( + "the destination leader holds the redo, and the replica stored the read result", + HOLD_DEADLINE, + Duration::from_millis(20), + || { + redo_parked.exists() + && !read_entries(node, dest_vshard).is_empty() + && stored_barrier_rows(node, dest_vshard) >= 1 + }, + ) + .await; + } + // The read-result entries the replica applied before it stops. The + // first is the one the barrier counts. + let applied_before = read_entries(&mv.cluster().nodes[restarted], dest_vshard); + let pinned = applied_before + .iter() + .map(|(_, index)| *index) + .max() + .expect("the replica applied a read result"); + filler + .simple_query(&filler_sql) + .await + .unwrap_or_else(|e| panic!("the filler write commits: {}", db_detail(&e))); + let dest_leader_applied = { + let nodes = &mv.cluster().nodes; + let leader = leader_of(&nodes[coordinator], dest_group); + nodes + .iter() + .find(|node| node.node_id == leader) + .expect("the destination leader runs") + .shared + .applied_index_watcher(dest_group) + .current() + }; + { + let node = &mv.cluster().nodes[restarted]; + wait_for( + "the replica's durable applied floor passes the read result's entry", + HOLD_DEADLINE, + Duration::from_millis(20), + || node.shared.pitr.durable_applied(dest_group) >= dest_leader_applied.max(pinned), + ) + .await; + } + + let stopped = mv + .fx + .cluster + .stop_member(restarted) + .await + .expect("stop the replica"); + mv.fx + .cluster + .restart_member(stopped) + .await + .expect("restart the replica"); + { + let node = &mv.cluster().nodes[restarted]; + assert_eq!(node.node_id, restarted_id, "the replica keeps its index"); + // Boot resumes delivery above the durable floor, which passed + // every entry applied before the stop. A duplicate read-result + // entry the log holds later can apply now: it sits at another + // index, and the barrier counts the first copy alone. + let again: Vec<(u64, u64)> = read_entries(node, dest_vshard) + .into_iter() + .filter(|entry| applied_before.contains(entry)) + .collect(); + assert!( + again.is_empty(), + "the restart delivers again the read-result entries {again:?}" + ); + assert_eq!( + stored_barrier_rows(node, dest_vshard), + 1, + "the stored row survives the restart" + ); + } + mv.cluster() + .transfer_leadership(dest_group, restarted_id) + .await; + std::fs::write(&redo_release, b"release").expect("release the redo"); + }); + moved.unwrap_or_else(|e| { + panic!( + "the move commits under the restarted leader: {}", + db_detail(&e) + ) + }); + drop(redo_gate); + + mv.wait_replicas_agree(Some(&source_bytes)).await; + wait_for( + "every node drops the barrier entries and rows of the finished move", + HOLD_DEADLINE, + Duration::from_millis(20), + || { + mv.cluster().nodes.iter().all(|node| { + waiting_barrier_txns(node, dest_vshard) == 0 + && stored_barrier_rows(node, dest_vshard) == 0 + }) + }, + ) + .await; + assert_no_apply_halt(mv.cluster()); + + let ItemMove { fx, .. } = mv; + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_timeout.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_timeout.rs new file mode 100644 index 000000000..b997a493a --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_timeout.rs @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A dependent-read barrier whose read result never arrives aborts on every +//! replica alike, through the timeout entry its leader puts in the log. +//! +//! 1. The fail gate `calvin::before_read_result_propose::` holds the +//! source vShard's broadcast of the item it read. +//! 2. The active vShards' barriers wait past +//! `tuning.calvin.dependent_read_passive_timeout_ms`. Each data-group +//! leader proposes the barrier's timeout entry, and every replica folds +//! it ahead of the read result. +//! 3. The move fails. No replica moved the item, and no scheduler halted. +//! 4. Released, the stale broadcast reaches both logs after the txn +//! finished. Every replica applies it and drops it, so no buffer and no +//! stored row keeps it. +//! 5. A second move commits: the aborted txn left no lock behind. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::time::Duration; + +use nodedb_types::fail_point::{FailAction, FailGuard}; + +use super::calvin_dependent_read_fixture::{ + ItemMove, assert_no_apply_halt, reads_applied, stored_barrier_rows, waiting_barrier_txns, +}; +use super::calvin_replica_content::strict_session; +use crate::common::cluster_harness::shared_steps::db_detail; +use crate::common::cluster_harness::wait_for; + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_read_result_held_past_the_timeout_aborts_on_every_replica() { + let mv = ItemMove::spawn("dep_timeout").await; + let source_bytes = mv.source_bytes().await; + let session = strict_session(mv.fx.coordinator()).await; + + let gate_dir = tempfile::tempdir().expect("gate directory"); + let release = gate_dir.path().join("release"); + let _gate = FailGuard::install( + &format!("calvin::before_read_result_propose::{}", mv.source), + FailAction::WaitForFile(release.clone()), + ); + + let aborted = session + .simple_query(&mv.transfer_sql()) + .await + .expect_err("the move aborts once its barrier times out in the log"); + assert!( + gate_dir.path().join("release.parked").exists(), + "the source vShard read the item and held its broadcast: {}", + db_detail(&aborted) + ); + + mv.wait_replicas_agree(None).await; + assert_no_apply_halt(mv.cluster()); + + let source_group = mv.group_of(&mv.source); + let dest_group = mv.group_of(&mv.dest); + let source_vshard = mv.source_vshard(); + let dest_vshard = mv.dest_vshard(); + for node in &mv.cluster().nodes { + assert_eq!( + reads_applied(node, source_vshard) + reads_applied(node, dest_vshard), + 0, + "node {} applied no read result while the gate held the broadcast", + node.node_id + ); + } + std::fs::write(&release, b"release").expect("release the held broadcast"); + // The released broadcast reaches both data-group logs. Every replica of + // each group applies its entry for the finished txn and keeps neither a + // buffered event nor a row of it. + wait_for( + "every replica applies the stale broadcast and drops it", + Duration::from_secs(30), + Duration::from_millis(50), + || { + mv.cluster().nodes.iter().all(|node| { + let source_done = !node.replicates_data_group(source_group) + || reads_applied(node, source_vshard) >= 1; + let dest_done = !node.replicates_data_group(dest_group) + || reads_applied(node, dest_vshard) >= 1; + source_done + && dest_done + && [source_vshard, dest_vshard].iter().all(|vshard| { + waiting_barrier_txns(node, *vshard) == 0 + && stored_barrier_rows(node, *vshard) == 0 + }) + }) + }, + ) + .await; + + session + .simple_query(&mv.transfer_sql()) + .await + .unwrap_or_else(|e| panic!("the second move commits: {}", db_detail(&e))); + mv.wait_replicas_agree(Some(&source_bytes)).await; + assert_no_apply_halt(mv.cluster()); + + let ItemMove { fx, .. } = mv; + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs index 5a1b58784..a246c9eda 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs @@ -15,6 +15,12 @@ mod calvin_backup_cut_order; mod calvin_cdc_net_kinds; mod calvin_cluster_array_read_set; mod calvin_cluster_pgwire_e2e; +mod calvin_dependent_read_commit; +mod calvin_dependent_read_early_result; +mod calvin_dependent_read_fixture; +mod calvin_dependent_read_follower_promotion; +mod calvin_dependent_read_restart; +mod calvin_dependent_read_timeout; mod calvin_duplicate_redo_copy; mod calvin_joiner_after_sequencer_compaction; mod calvin_leader_change_mid_txn; diff --git a/nodedb-cluster/src/calvin/sequencer/inbox.rs b/nodedb-cluster/src/calvin/sequencer/inbox.rs index 50d9ab910..a36e76d96 100644 --- a/nodedb-cluster/src/calvin/sequencer/inbox.rs +++ b/nodedb-cluster/src/calvin/sequencer/inbox.rs @@ -740,6 +740,7 @@ mod tests { ); m }, + expected: BTreeMap::new(), }); let err = inbox.submit(tx).expect_err("should be rejected"); @@ -782,6 +783,7 @@ mod tests { } m }, + expected: BTreeMap::new(), }); let err = inbox.submit(tx).expect_err("should be rejected"); diff --git a/nodedb-cluster/src/calvin/types/mod.rs b/nodedb-cluster/src/calvin/types/mod.rs index 18dcfde82..c4bd86d01 100644 --- a/nodedb-cluster/src/calvin/types/mod.rs +++ b/nodedb-cluster/src/calvin/types/mod.rs @@ -15,8 +15,8 @@ pub use multi_part::{ MultiPartPlans, PartStreamId, PlanPart, StreamedPart, TaskChunk, VShardParts, }; pub use primitives::{ - DependentReadSpec, EngineKeySet, EngineTag, PassiveReadKey, ReadKeyIdent, SortedVec, - VersionedReadEntry, VersionedReadSet, + DependentReadSpec, EngineKeySet, EngineTag, PassiveKey, PassiveReadKey, PassiveReadKeyId, + ReadKeyIdent, SortedVec, VersionedReadEntry, VersionedReadSet, }; pub use read_write_set::ReadWriteSet; pub use scheduler_input::SchedulerInput; diff --git a/nodedb-cluster/src/calvin/types/primitives.rs b/nodedb-cluster/src/calvin/types/primitives.rs index 5954a6ea7..8f1103ee9 100644 --- a/nodedb-cluster/src/calvin/types/primitives.rs +++ b/nodedb-cluster/src/calvin/types/primitives.rs @@ -12,14 +12,19 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; pub use nodedb_types::calvin::{ - EngineKeySet, EngineTag, PassiveReadKey, ReadKeyIdent, SortedVec, VersionedReadEntry, - VersionedReadSet, + EngineKeySet, EngineTag, PassiveKey, PassiveReadKey, PassiveReadKeyId, ReadKeyIdent, SortedVec, + VersionedReadEntry, VersionedReadSet, }; /// Describes the passive-read participants for a dependent-read Calvin txn. /// -/// Each entry maps a vshard id to the keys that vshard must read and broadcast -/// to all active participants before any writes can proceed. +/// `passive_reads` maps each passive vShard to the keys it reads at its lock +/// grant and broadcasts to every active participant before any write stages. +/// +/// `expected` holds the value the coordinator read for each passive row +/// before it planned the writes: the stored bytes, `None` for an absent row. +/// An active participant whose broadcast values differ votes +/// `PredictionDrift`, and the coordinator reads again. /// /// `BTreeMap` is mandatory here: the sequencer and scheduler must iterate /// vshards in a deterministic order (determinism contract). @@ -36,20 +41,35 @@ pub use nodedb_types::calvin::{ pub struct DependentReadSpec { /// Passive participants: vshard → keys to read. pub passive_reads: BTreeMap>, + /// The value the coordinator read for each passive row. + pub expected: BTreeMap>>, } impl DependentReadSpec { - /// Total estimated serialized bytes across all passive read keys. + /// Total estimated serialized bytes across all passive read keys and + /// the expected values the coordinator read. /// /// Used by the sequencer admission check to enforce /// `max_dependent_read_bytes_per_txn`. This is an O(1)-per-key /// estimate, not an exact serialized size. pub fn total_bytes(&self) -> usize { - self.passive_reads + let keys: usize = self + .passive_reads .values() .flat_map(|ks| ks.iter()) .map(|k| k.engine_key.serialized_size_hint()) - .sum() + .sum(); + let expected: usize = self + .expected + .iter() + .map(|(id, value)| id.serialized_size_hint() + value.as_ref().map_or(0, Vec::len)) + .sum(); + keys + expected + } + + /// The passive vShards, in id order. + pub fn passive_vshards(&self) -> impl Iterator + '_ { + self.passive_reads.keys().copied() } } @@ -160,10 +180,39 @@ mod tests { ); m }, + expected: BTreeMap::from([( + PassiveReadKeyId::kv( + nodedb_types::QualifiedCollection::from_stored("sessions".to_owned()), + b"abc".to_vec(), + ), + Some(b"stored".to_vec()), + )]), }; let bytes = zerompk::to_msgpack_vec(&spec).unwrap(); let decoded: DependentReadSpec = zerompk::from_msgpack(&bytes).unwrap(); - assert_eq!(spec.passive_reads.len(), decoded.passive_reads.len()); - assert_eq!(spec.passive_reads.get(&1), decoded.passive_reads.get(&1)); + assert_eq!(decoded, spec); + } + + /// The byte estimate counts the expected values the coordinator read. + #[test] + fn total_bytes_counts_expected_values() { + let mut spec = DependentReadSpec { + passive_reads: BTreeMap::from([( + 1u32, + vec![PassiveReadKey { + engine_key: kv_set("s", vec![b"k".to_vec()]), + }], + )]), + expected: BTreeMap::new(), + }; + let keys_only = spec.total_bytes(); + spec.expected.insert( + PassiveReadKeyId::kv( + nodedb_types::QualifiedCollection::from_stored("s".to_owned()), + b"k".to_vec(), + ), + Some(vec![0u8; 100]), + ); + assert!(spec.total_bytes() >= keys_only + 100); } } diff --git a/nodedb-cluster/src/calvin/types/transaction.rs b/nodedb-cluster/src/calvin/types/transaction.rs index 9d68d2b9f..4585a1d38 100644 --- a/nodedb-cluster/src/calvin/types/transaction.rs +++ b/nodedb-cluster/src/calvin/types/transaction.rs @@ -347,28 +347,39 @@ impl TxClass { Ok(participants) } - /// Ergonomic constructor for dependent-read Calvin transactions. + /// Constructor for a dependent-read Calvin transaction of `database_id`. /// - /// Equivalent to `TxClass::new(read_set, write_set, plans, tenant_id, - /// Some(dependent_reads), versioned_reads)`. + /// The write set spans two or more vShards, as [`Self::new_in_database`] + /// requires. Every passive vShard of `dependent_reads` participates. pub fn new_dependent( read_set: ReadWriteSet, write_set: ReadWriteSet, plans: Vec, tenant_id: TenantId, + database_id: DatabaseId, dependent_reads: DependentReadSpec, versioned_reads: VersionedReadSet, ) -> Result { - Self::new( + Self::new_in_database( read_set, write_set, plans, tenant_id, + database_id, Some(dependent_reads), versioned_reads, ) } + /// The vShards the write set names: the active participants of a + /// dependent-read transaction. + /// + /// Fails when a write-set collection name lacks the qualifier of the + /// class's database. Construction rejects such a class. + pub fn active_vshards(&self) -> Result, CalvinError> { + self.write_vshards() + } + /// The vShards that must receive this transaction's slice. /// /// Derived from the write set's collection names. Re-derived after @@ -734,6 +745,7 @@ mod tests { ); m }, + expected: std::collections::BTreeMap::new(), }; let tc = TxClass::new( diff --git a/nodedb-cluster/src/raft_loop/proposals.rs b/nodedb-cluster/src/raft_loop/proposals.rs index 8bb2a28da..d2076c66c 100644 --- a/nodedb-cluster/src/raft_loop/proposals.rs +++ b/nodedb-cluster/src/raft_loop/proposals.rs @@ -9,11 +9,6 @@ use crate::error::Result; use super::loop_core::{CommitApplier, RaftLoop}; use crate::forward::PlanExecutor; -/// How long a forwarded data propose waits for the leader's reply past the -/// proposer's deadline. The leader's gate stops at that deadline, so the -/// margin covers the round trip and the leader's verdict arrives. -const FORWARD_REPLY_MARGIN: std::time::Duration = std::time::Duration::from_secs(2); - impl RaftLoop { /// Propose a command to the Raft group owning the given vShard. /// @@ -233,7 +228,7 @@ impl RaftLoop { /// as `ClusterError::Calvin`. The gate waits no later than `deadline`, /// the caller's deadline. A forward carries what remains of it. /// - /// The call ends no later than `deadline` plus [`FORWARD_REPLY_MARGIN`]. + /// The call ends no later than `deadline` plus [`crate::rpc_codec::REPLY_MARGIN`]. /// `CalvinError::AdmissionTimedOut` means the entry was not proposed. /// `ClusterError::ShardTimeout` means the leader's reply never came, so /// the leader can have proposed the entry. `ClusterError::Unanswered` @@ -283,7 +278,7 @@ impl RaftLoop { /// Send a `DataProposeRequest` carrying the budget left to `deadline` to /// `leader_id`, and wait for its verdict until `deadline` plus - /// [`FORWARD_REPLY_MARGIN`]. + /// [`crate::rpc_codec::REPLY_MARGIN`]. async fn forward_data_propose( &self, leader_id: u64, @@ -328,8 +323,7 @@ impl RaftLoop { }); // The generic short RPC timeout ends the call before a contended // write's gate wait does. The reply wait follows the forwarded budget. - let reply_budget = - std::time::Duration::from_millis(deadline_remaining_ms) + FORWARD_REPLY_MARGIN; + let reply_budget = crate::rpc_codec::reply_wait(deadline_remaining_ms); let sent = tokio::time::timeout( reply_budget, self.transport diff --git a/nodedb-cluster/src/rpc_codec/data_propose.rs b/nodedb-cluster/src/rpc_codec/data_propose.rs index 63ba4c3cc..6bd70dae7 100644 --- a/nodedb-cluster/src/rpc_codec/data_propose.rs +++ b/nodedb-cluster/src/rpc_codec/data_propose.rs @@ -38,6 +38,21 @@ pub fn remaining_budget_ms(deadline: tokio::time::Instant) -> u64 { u64::try_from(remaining.as_millis()).unwrap_or(u64::MAX) } +/// How long a caller waits for a remote handler's reply past the budget the +/// request carries. The handler stops at that budget, so the margin covers +/// the round trip and its verdict arrives. +pub const REPLY_MARGIN: std::time::Duration = std::time::Duration::from_secs(2); + +/// How long a caller waits for the reply to a request that carries +/// `deadline_remaining_ms`: the request's own budget plus [`REPLY_MARGIN`]. +/// +/// A handler that can wait (a write gate, a Calvin queue) answers within the +/// budget it received. The transport's fixed RPC timeout never bounds such a +/// reply: a statement deadline longer than it would end early. +pub fn reply_wait(deadline_remaining_ms: u64) -> std::time::Duration { + std::time::Duration::from_millis(deadline_remaining_ms).saturating_add(REPLY_MARGIN) +} + /// The leader's deadline for a forward that carries `deadline_remaining_ms`. /// `None` when the budget is spent. /// @@ -261,6 +276,21 @@ pub(super) fn decode_data_propose_resp(payload: &[u8]) -> Result { mod tests { use super::*; use crate::cluster_epoch::ClusterEpochState; + + /// A reply wait follows the request's own budget, past any fixed RPC + /// timeout, plus the margin. + #[test] + fn a_reply_wait_follows_the_request_budget() { + assert_eq!( + reply_wait(30_000), + std::time::Duration::from_millis(30_000) + REPLY_MARGIN + ); + assert_eq!(reply_wait(0), REPLY_MARGIN); + assert_eq!( + reply_wait(u64::MAX), + std::time::Duration::from_millis(u64::MAX) + REPLY_MARGIN + ); + } use crate::rpc_codec::{decode, encode}; fn roundtrip(target: ProposeTarget) -> DataProposeRequest { diff --git a/nodedb-cluster/src/rpc_codec/mod.rs b/nodedb-cluster/src/rpc_codec/mod.rs index aab00dcab..e75b05d7c 100644 --- a/nodedb-cluster/src/rpc_codec/mod.rs +++ b/nodedb-cluster/src/rpc_codec/mod.rs @@ -54,8 +54,8 @@ pub use data_plane_error::{ DataPlaneCounterFault, DataPlaneErrorCode, DataPlaneSyncHold, DataPlaneTextColumnFault, }; pub use data_propose::{ - DataProposeRequest, DataProposeResponse, ForwardedProposeRefusal, ProposeTarget, - forwarded_deadline, remaining_budget_ms, + DataProposeRequest, DataProposeResponse, ForwardedProposeRefusal, ProposeTarget, REPLY_MARGIN, + forwarded_deadline, remaining_budget_ms, reply_wait, }; pub use execute::{ DescriptorVersionEntry, ExecuteRequest, ExecuteResponse, ExecuteStreamChunk, ExecuteStreamEnd, diff --git a/nodedb-physical/src/physical_plan/meta.rs b/nodedb-physical/src/physical_plan/meta.rs index 8fafaa00f..75853a614 100644 --- a/nodedb-physical/src/physical_plan/meta.rs +++ b/nodedb-physical/src/physical_plan/meta.rs @@ -8,7 +8,7 @@ use nodedb_types::calvin::{PassiveReadKey, VersionedReadEntry}; use nodedb_types::timeseries::continuous_agg::ContinuousAggregateDef; use nodedb_types::{QualifiedCollection, TenantId, Value}; -pub use super::meta_calvin::PassiveReadKeyId; +pub use super::meta_calvin::{PassiveKey, PassiveReadKeyId}; /// Meta / maintenance physical operations. #[derive( @@ -358,15 +358,16 @@ pub enum MetaOp { body_plans: Vec, }, - /// Calvin dependent-read executor: passive participant reads keys and - /// returns values for broadcast. + /// Calvin dependent-read executor: a passive participant reads its keys + /// and returns their values for broadcast. /// - /// Dispatched by the scheduler to passive vshards (those holding only - /// read keys, not write keys) for a dependent-read Calvin transaction. - /// The Data Plane handler reads each key from the local engine and - /// returns a msgpack-encoded `Vec<(PassiveReadKeyId, Value)>` payload. - /// The scheduler then proposes a `ReplicatedWrite::CalvinReadResult` - /// to the per-vshard Raft group so all replicas see the same values. + /// The data-group leader of each passive vShard dispatches it once the + /// transaction holds its locks there. The Data Plane handler reads each + /// key from base storage and returns a zerompk-encoded + /// `Vec<(PassiveReadKeyId, Value)>`: the stored bytes as `Value::Bytes`, + /// `Value::Null` for an absent row. A successful read votes commit. The + /// scheduler proposes the payload as a `ReplicatedWrite::CalvinReadResult` + /// to the data group of every active vShard. CalvinExecutePassive { /// Sequencer epoch this transaction belongs to. epoch: u64, diff --git a/nodedb-physical/src/physical_plan/meta_calvin.rs b/nodedb-physical/src/physical_plan/meta_calvin.rs index 8fed613b7..2e53bcbcb 100644 --- a/nodedb-physical/src/physical_plan/meta_calvin.rs +++ b/nodedb-physical/src/physical_plan/meta_calvin.rs @@ -1,34 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 //! Calvin-specific identity types carried by `MetaOp` Calvin variants. +//! +//! [`PassiveReadKeyId`] keys `MetaOp::CalvinExecuteActive::injected_reads`. +//! It lives in `nodedb-types`, so the replicated transaction class in +//! `nodedb-cluster` names the same identity. -/// Identity of a single key read by a passive Calvin participant. -/// -/// Used as the map key in `MetaOp::CalvinExecuteActive::injected_reads` so -/// active participants can look up which value belongs to which key. -/// -/// `collection` is database-qualified (`QualifiedCollection`) so a key from -/// one database never collides with the same bare name in another. -/// -/// `BTreeMap` key: `Ord` is derived lexicographically — collection first, -/// surrogate second. This is the determinism contract: all replicas must -/// iterate `injected_reads` in the same order. -#[derive( - Debug, - Clone, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - serde::Serialize, - serde::Deserialize, - zerompk::ToMessagePack, - zerompk::FromMessagePack, -)] -pub struct PassiveReadKeyId { - /// Collection the key belongs to. - pub collection: nodedb_types::QualifiedCollection, - /// Global surrogate for the row. - pub surrogate: u32, -} +pub use nodedb_types::calvin::{PassiveKey, PassiveReadKeyId}; diff --git a/nodedb-types/src/calvin.rs b/nodedb-types/src/calvin.rs index 5b214bfe0..e8d44d6c7 100644 --- a/nodedb-types/src/calvin.rs +++ b/nodedb-types/src/calvin.rs @@ -12,6 +12,8 @@ use serde::{Deserialize, Serialize}; use crate::{KeyRepr, WriteVersion}; +pub use crate::calvin_passive::{PassiveKey, PassiveReadKeyId}; + /// A newtype over `Vec` that guarantees sorted, deduplicated contents. /// /// Constructed via [`SortedVec::new`], which sorts and deduplicates at @@ -202,13 +204,12 @@ impl EngineKeySet { } } -/// A single key that a passive participant must read and broadcast. +/// A key set a passive participant reads and broadcasts. /// -/// Wraps an [`EngineKeySet`]; per the dependent-read protocol each -/// `PassiveReadKey` contains a single-element (or small) key set. The -/// sequencer does not enforce single-element sets; the scheduler enforces the -/// total byte budget via `DependentReadSpec::total_bytes()` (which lives in -/// `nodedb-cluster`). +/// Wraps an [`EngineKeySet`]. A passive vShard reads document rows by +/// surrogate and key-value rows by key. It refuses every other key set. The +/// sequencer bounds the bytes a transaction reads passively through +/// `DependentReadSpec::total_bytes()` in `nodedb-cluster`. #[derive( Debug, Clone, diff --git a/nodedb-types/src/calvin_passive.rs b/nodedb-types/src/calvin_passive.rs new file mode 100644 index 000000000..a777120ca --- /dev/null +++ b/nodedb-types/src/calvin_passive.rs @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Identity of one row a passive Calvin participant reads. +//! +//! A dependent-read transaction names the rows its passive vShards read. Each +//! passive vShard reads them at its lock grant and broadcasts one value per +//! row through the data-group log of every active vShard. The active vShard +//! matches each value to its row by [`PassiveReadKeyId`]. +//! +//! The identity is exact: a key-value row keeps its key bytes, so two keys +//! never share an identity. + +use serde::{Deserialize, Serialize}; + +use crate::QualifiedCollection; + +/// The row within a collection a passive read names. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub enum PassiveKey { + /// A document row, by its surrogate. + Surrogate { surrogate: u32 }, + /// A key-value row, by its key bytes. + Kv { key: Vec }, +} + +/// Identity of one row a passive participant reads. +/// +/// The map key of every passive read value. `Ord` is derived, collection +/// first, so every replica iterates a map of them in one order. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct PassiveReadKeyId { + /// The database-qualified collection of the row. + pub collection: QualifiedCollection, + /// The row within the collection. + pub key: PassiveKey, +} + +impl PassiveReadKeyId { + /// The identity of document row `surrogate` of `collection`. + pub fn surrogate(collection: QualifiedCollection, surrogate: u32) -> Self { + Self { + collection, + key: PassiveKey::Surrogate { surrogate }, + } + } + + /// The identity of key-value row `key` of `collection`. + pub fn kv(collection: QualifiedCollection, key: Vec) -> Self { + Self { + collection, + key: PassiveKey::Kv { key }, + } + } + + /// Bytes this identity adds to a replicated entry, estimated. + pub fn serialized_size_hint(&self) -> usize { + let key = match &self.key { + PassiveKey::Surrogate { .. } => 4, + PassiveKey::Kv { key } => key.len(), + }; + self.collection.as_str().len() + key + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Two key-value rows keep two identities, whatever their keys hash to. + #[test] + fn kv_rows_keep_distinct_identities() { + let collection = QualifiedCollection::from_stored("items".to_owned()); + let a = PassiveReadKeyId::kv(collection.clone(), b"alice:sword".to_vec()); + let b = PassiveReadKeyId::kv(collection, b"alice:shield".to_vec()); + assert_ne!(a, b); + } + + /// An identity survives a msgpack round trip. + #[test] + fn identity_round_trips_through_msgpack() { + let id = PassiveReadKeyId::surrogate(QualifiedCollection::from_stored("o".to_owned()), 9); + let bytes = zerompk::to_msgpack_vec(&id).expect("encode"); + let back: PassiveReadKeyId = zerompk::from_msgpack(&bytes).expect("decode"); + assert_eq!(back, id); + } +} diff --git a/nodedb-types/src/config/tuning/calvin.rs b/nodedb-types/src/config/tuning/calvin.rs index 13ac0e585..03c5e1faf 100644 --- a/nodedb-types/src/config/tuning/calvin.rs +++ b/nodedb-types/src/config/tuning/calvin.rs @@ -17,8 +17,11 @@ fn default_txn_deadline_multiplier() -> u32 { } fn default_dependent_read_passive_timeout_ms() -> u64 { - // Three 20 ms sequencer epochs. - 60 + // A passive read crosses two data groups: its read result is proposed to + // each active vShard's group, often led by another node. The timeout only + // bounds how long a lost result holds the active vShard's locks, so it + // leaves room for a leader change on the way. 250 sequencer epochs. + 5_000 } fn default_verdict_stall_warn_ms() -> u64 { @@ -60,8 +63,8 @@ fn default_restage_backoff_ms() -> u64 { /// Tuning knobs for the Calvin scheduler that runs per hosted vShard. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CalvinTuning { - /// Capacity of each scheduler's bounded input, completion, read-result, - /// and verdict channels. + /// Capacity of each scheduler's bounded input, completion, and verdict + /// channels. #[serde(default = "default_channel_capacity")] pub channel_capacity: usize, @@ -69,8 +72,10 @@ pub struct CalvinTuning { #[serde(default = "default_txn_deadline_multiplier")] pub txn_deadline_multiplier: u32, - /// Timeout in milliseconds for passive participant read results of a - /// dependent-read transaction. + /// Wait in milliseconds of an active vShard's dependent-read barrier for + /// the read results of its passive vShards. Past it, the vShard's + /// data-group leader proposes the barrier's timeout entry, and every + /// replica aborts a barrier that entry finds incomplete. #[serde(default = "default_dependent_read_passive_timeout_ms")] pub dependent_read_passive_timeout_ms: u64, @@ -82,7 +87,9 @@ pub struct CalvinTuning { /// In-flight backlog at which a scheduler stops taking new sequenced /// input. The backlog counts pending, blocked, and dependent-barrier - /// transactions. + /// transactions. It also bounds the dependent-read barrier entries a + /// vShard holds in memory for transactions its scheduler has not taken + /// yet. Entries past it wait in the system catalog alone. #[serde(default = "default_max_inflight_backlog")] pub max_inflight_backlog: usize, @@ -141,7 +148,7 @@ mod tests { let t = CalvinTuning::default(); assert_eq!(t.channel_capacity, 512); assert_eq!(t.txn_deadline_multiplier, 3); - assert_eq!(t.dependent_read_passive_timeout_ms, 60); + assert_eq!(t.dependent_read_passive_timeout_ms, 5_000); assert_eq!(t.verdict_stall_warn_ms, 5_000); assert_eq!(t.max_inflight_backlog, 1_024); assert_eq!(t.catch_up_window, 512); diff --git a/nodedb-types/src/lib.rs b/nodedb-types/src/lib.rs index 64c83b675..13b965607 100644 --- a/nodedb-types/src/lib.rs +++ b/nodedb-types/src/lib.rs @@ -19,6 +19,7 @@ pub mod ascii; pub mod backup_envelope; pub mod bbox; pub mod calvin; +pub mod calvin_passive; pub mod clone; pub mod collection; pub mod collection_config; diff --git a/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs b/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs index 36f7ebc6f..b594003c9 100644 --- a/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs +++ b/nodedb/src/control/cluster/array_cluster_exec/dispatch.rs @@ -130,9 +130,13 @@ impl NexarArrayDispatch { // Wrap in RaftRpc and send. The remote node's handler decodes the // envelope bytes and dispatches to the appropriate shard handler. let rpc = RaftRpc::VShardEnvelope(envelope_bytes); + // The shard's budget bounds the reply wait, never the transport's fixed + // RPC timeout. + let reply_wait = std::time::Duration::from_millis(timeout_ms); let resp_rpc = tokio::time::timeout( - std::time::Duration::from_millis(timeout_ms), - self.transport.send_rpc(node_id, rpc), + reply_wait, + self.transport + .send_rpc_with_read_timeout(node_id, rpc, reply_wait), ) .await // The request can have reached the shard, so a timeout is never diff --git a/nodedb/src/control/cluster/calvin/mod.rs b/nodedb/src/control/cluster/calvin/mod.rs index 13c67536a..1f2051c19 100644 --- a/nodedb/src/control/cluster/calvin/mod.rs +++ b/nodedb/src/control/cluster/calvin/mod.rs @@ -5,6 +5,6 @@ pub mod scheduler; pub use executor::{OllpConfig, OllpError, OllpOrchestrator}; pub use scheduler::{ - CalvinReadResultProposal, RaftSequencerProposer, ReadResultEvent, Scheduler, SchedulerConfig, + BarrierEvent, CalvinReadResultProposal, RaftSequencerProposer, Scheduler, SchedulerConfig, SchedulerParams, SequencerProposer, propose_calvin_read_result, }; diff --git a/nodedb/src/control/cluster/calvin/scheduler/barrier_log.rs b/nodedb/src/control/cluster/calvin/scheduler/barrier_log.rs new file mode 100644 index 000000000..5b8c6223d --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/barrier_log.rs @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The dependent-read barrier events of one txn on one vShard, folded in the +//! order the vShard's data-group log holds them. +//! +//! Two kinds of entry reach a barrier through the log: a passive vShard's +//! read result, and the barrier's timeout. Every replica of the vShard +//! applies the log in one order, so every replica folds the same events into +//! the same [`BarrierLog`]: +//! +//! - A read result below the timeout entry counts. One above it does not. +//! - The first read result of a passive vShard counts. A later copy, from a +//! re-proposal, does not. +//! +//! [`BarrierLog::outcome`] is then the same on every replica: the barrier +//! completes when every passive vShard's result counts, and it times out +//! when the timeout entry came first. + +use std::collections::{BTreeMap, BTreeSet}; + +use nodedb_physical::physical_plan::meta::PassiveReadKeyId; +use nodedb_types::Value; + +/// Read values received from passive vShards, keyed by passive vShard. +/// `BTreeMap` for determinism. +pub type ReceivedReads = BTreeMap>; + +/// One barrier entry of the data-group log. +#[derive(Debug, Clone, PartialEq)] +pub enum BarrierEvent { + /// A passive vShard's read values. + Read { + passive_vshard: u32, + values: Vec<(PassiveReadKeyId, Value)>, + }, + /// The barrier's timeout. + Timeout, +} + +/// Where a barrier stands once its log is folded. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BarrierOutcome { + /// Every passive vShard's result came before any timeout. + Complete, + /// The timeout came before a passive vShard's result. + TimedOut, + /// A passive vShard's result is still missing, and no timeout came. + Waiting, +} + +/// The barrier events of one txn, folded in log order. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct BarrierLog { + reads: ReceivedReads, + timed_out: bool, +} + +impl BarrierLog { + /// Whether folding `event` changes this log: a timeout, or the first + /// result of a passive vShard, before any timeout. + pub fn accepts(&self, event: &BarrierEvent) -> bool { + if self.timed_out { + return false; + } + match event { + BarrierEvent::Read { passive_vshard, .. } => !self.reads.contains_key(passive_vshard), + BarrierEvent::Timeout => true, + } + } + + /// Fold `event`, the next barrier entry of the log. + pub fn note(&mut self, event: BarrierEvent) { + if !self.accepts(&event) { + return; + } + match event { + BarrierEvent::Read { + passive_vshard, + values, + } => { + self.reads.insert(passive_vshard, values); + } + BarrierEvent::Timeout => self.timed_out = true, + } + } + + /// Fold `later`, the events the log holds after every event of this one. + pub fn extend(&mut self, later: BarrierLog) { + if self.timed_out { + return; + } + for (passive_vshard, values) in later.reads { + self.reads.entry(passive_vshard).or_insert(values); + } + self.timed_out = later.timed_out; + } + + /// The events this log holds: one per counted result, one for a timeout. + pub fn event_count(&self) -> usize { + self.reads.len() + usize::from(self.timed_out) + } + + /// Where a barrier that waits for every vShard of `passive` stands. + pub fn outcome(&self, passive: &BTreeSet) -> BarrierOutcome { + if passive.iter().all(|vshard| self.reads.contains_key(vshard)) { + BarrierOutcome::Complete + } else if self.timed_out { + BarrierOutcome::TimedOut + } else { + BarrierOutcome::Waiting + } + } + + /// The vShards of `passive` whose result has not counted. + pub fn missing(&self, passive: &BTreeSet) -> Vec { + passive + .iter() + .copied() + .filter(|vshard| !self.reads.contains_key(vshard)) + .collect() + } + + /// Every counted read value, keyed by the row it names. `BTreeMap` + /// iterates in one order on every replica. + pub fn injected_reads(&self) -> BTreeMap { + self.reads + .values() + .flatten() + .map(|(key_id, value)| (key_id.clone(), value.clone())) + .collect() + } + + /// The log as the bytes a stored barrier row holds. + pub fn to_bytes(&self) -> crate::Result> { + let stored = StoredBarrierLog { + reads: self + .reads + .iter() + .map(|(vshard, values)| (*vshard, values.clone())) + .collect(), + timed_out: self.timed_out, + }; + zerompk::to_msgpack_vec(&stored).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("encode calvin barrier log: {e}"), + }) + } + + /// The log a stored barrier row holds. + pub fn from_bytes(bytes: &[u8]) -> crate::Result { + let stored: StoredBarrierLog = + zerompk::from_msgpack(bytes).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("decode calvin barrier log: {e}"), + })?; + Ok(Self { + reads: stored.reads.into_iter().collect(), + timed_out: stored.timed_out, + }) + } +} + +/// The stored form of a [`BarrierLog`]. +#[derive(zerompk::ToMessagePack, zerompk::FromMessagePack)] +#[msgpack(map)] +struct StoredBarrierLog { + reads: Vec<(u32, Vec<(PassiveReadKeyId, Value)>)>, + timed_out: bool, +} + +#[cfg(test)] +mod tests { + use nodedb_types::QualifiedCollection; + + use super::*; + + fn read(passive_vshard: u32, value: i64) -> BarrierEvent { + BarrierEvent::Read { + passive_vshard, + values: vec![( + PassiveReadKeyId::surrogate( + QualifiedCollection::from_stored(format!("c{passive_vshard}")), + 1, + ), + Value::Integer(value), + )], + } + } + + fn passive(vshards: &[u32]) -> BTreeSet { + vshards.iter().copied().collect() + } + + /// Every result before the timeout completes the barrier: the timeout + /// that follows changes nothing. + #[test] + fn results_before_the_timeout_complete_the_barrier() { + let mut log = BarrierLog::default(); + log.note(read(1, 10)); + log.note(read(2, 20)); + log.note(BarrierEvent::Timeout); + assert_eq!(log.outcome(&passive(&[1, 2])), BarrierOutcome::Complete); + assert_eq!(log.injected_reads().len(), 2); + } + + /// A result above the timeout does not count, so the barrier times out. + #[test] + fn a_result_after_the_timeout_does_not_count() { + let mut log = BarrierLog::default(); + log.note(read(1, 10)); + log.note(BarrierEvent::Timeout); + log.note(read(2, 20)); + assert_eq!(log.outcome(&passive(&[1, 2])), BarrierOutcome::TimedOut); + assert_eq!(log.missing(&passive(&[1, 2])), vec![2]); + } + + /// A barrier missing a result with no timeout waits. + #[test] + fn a_missing_result_waits() { + let mut log = BarrierLog::default(); + log.note(read(1, 10)); + assert_eq!(log.outcome(&passive(&[1, 2])), BarrierOutcome::Waiting); + } + + /// The first copy of a passive vShard's result counts. A re-proposed + /// copy does not replace it. + #[test] + fn the_first_copy_of_a_result_counts() { + let mut log = BarrierLog::default(); + log.note(read(1, 10)); + assert!(!log.accepts(&read(1, 99))); + log.note(read(1, 99)); + let values: Vec = log.injected_reads().into_values().collect(); + assert_eq!(values, vec![Value::Integer(10)]); + } + + /// Two logs folded one after the other give the outcome one log of + /// every event gives. + #[test] + fn extending_with_later_events_keeps_log_order() { + let mut earlier = BarrierLog::default(); + earlier.note(read(1, 10)); + let mut later = BarrierLog::default(); + later.note(BarrierEvent::Timeout); + later.note(read(2, 20)); + earlier.extend(later); + assert_eq!(earlier.outcome(&passive(&[1, 2])), BarrierOutcome::TimedOut); + + let mut timed_out = BarrierLog::default(); + timed_out.note(BarrierEvent::Timeout); + let mut late_reads = BarrierLog::default(); + late_reads.note(read(1, 10)); + timed_out.extend(late_reads); + assert_eq!(timed_out.outcome(&passive(&[1])), BarrierOutcome::TimedOut); + } + + /// A stored log decodes to the log it was encoded from. + #[test] + fn a_stored_log_round_trips() { + let mut log = BarrierLog::default(); + log.note(read(2, 20)); + log.note(read(1, 10)); + log.note(BarrierEvent::Timeout); + let bytes = log.to_bytes().expect("encode"); + assert_eq!(BarrierLog::from_bytes(&bytes).expect("decode"), log); + } + + /// The event count is what the log holds: counted results and a timeout. + #[test] + fn event_count_counts_held_events() { + let mut log = BarrierLog::default(); + assert_eq!(log.event_count(), 0); + log.note(read(1, 10)); + log.note(read(1, 11)); + assert_eq!(log.event_count(), 1); + log.note(BarrierEvent::Timeout); + assert_eq!(log.event_count(), 2); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/barrier_store.rs b/nodedb/src/control/cluster/calvin/scheduler/barrier_store.rs new file mode 100644 index 000000000..ff200246d --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/barrier_store.rs @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The stored dependent-read barrier log of each txn on each vShard. +//! +//! The data-group apply loop concludes a barrier entry as durably applied +//! only once its row in `_system.calvin_barrier_logs` holds the event. The +//! durable applied floor of a group asserts that every entry at or below it +//! is durable, and boot resumes Raft delivery above it. So an entry below +//! the floor is never delivered again, and its row is what a barrier of the +//! txn reads after a restart: +//! +//! - [`restore_at_boot`] marks the txn of every stored row stored in its +//! vShard's buffer, and removes the rows of finished txns; +//! - [`capture_group`] and [`install_group`] carry the rows of a data group's +//! unfinished txns in its snapshot, for the entries the snapshot covers; +//! - the scheduler removes a txn's row once the txn finished, and +//! [`sweep_finished`] removes on its stall tick each finished txn's row +//! that removal missed. + +use std::collections::{BTreeMap, BTreeSet}; + +use super::applied_ledger::CalvinAppliedLedger; +use super::barrier_log::{BarrierEvent, BarrierLog}; +use super::lock::TxnId; +use crate::control::security::catalog::SystemCatalog; +use crate::control::security::catalog::calvin_barrier_logs::StoredBarrierLog; +use crate::control::state::SharedState; +use crate::types::CutBarrierLog; + +/// Fold `event` into the stored row of `txn` on `vshard_id`. An event the +/// row's log does not take, a copy it holds or one after its timeout, +/// writes nothing. +pub fn save_event( + catalog: &SystemCatalog, + vshard_id: u32, + txn: TxnId, + event: &BarrierEvent, +) -> crate::Result<()> { + catalog.fold_calvin_barrier_log(vshard_id, txn.epoch, txn.position, |held| { + let mut log = match held { + Some(bytes) => BarrierLog::from_bytes(bytes)?, + None => BarrierLog::default(), + }; + if held.is_some() && !log.accepts(event) { + return Ok(None); + } + log.note(event.clone()); + log.to_bytes().map(Some) + }) +} + +/// The stored log of `txn` on `vshard_id`. A txn with no row has an empty +/// log. +pub fn load_log(catalog: &SystemCatalog, vshard_id: u32, txn: TxnId) -> crate::Result { + match catalog.load_calvin_barrier_log(vshard_id, txn.epoch, txn.position)? { + Some(bytes) => BarrierLog::from_bytes(&bytes), + None => Ok(BarrierLog::default()), + } +} + +/// Remove the stored row of `txn` on `vshard_id`, which finished. +pub fn remove_log(catalog: &SystemCatalog, vshard_id: u32, txn: TxnId) -> crate::Result<()> { + catalog.remove_calvin_barrier_logs(&[(vshard_id, txn.epoch, txn.position)]) +} + +/// Remove every stored row of `vshard_id` whose txn `ledger` marks applied. +/// Returns how many rows went. +/// +/// A finished txn's row stays when its removal at the finish failed, or +/// when its event was saved while the txn finished. The scheduler runs this +/// on its stall tick, so such a row goes while the node runs, and a failed +/// pass retries on the next tick. +pub fn sweep_finished( + catalog: &SystemCatalog, + vshard_id: u32, + ledger: &CalvinAppliedLedger, +) -> crate::Result { + let finished: Vec<(u32, u64, u32)> = catalog + .load_calvin_barrier_logs(Some(&BTreeSet::from([vshard_id])))? + .into_iter() + .filter(|row| ledger.is_applied(row.epoch, row.position)) + .map(|row| (row.vshard_id, row.epoch, row.position)) + .collect(); + catalog.remove_calvin_barrier_logs(&finished)?; + Ok(finished.len()) +} + +/// Whether `shared`'s ledger of `vshard_id` holds `(epoch, position)`. +fn is_applied(shared: &SharedState, vshard_id: u32, epoch: u64, position: u32) -> bool { + shared + .calvin + .applied + .get(vshard_id) + .is_some_and(|ledger| ledger.is_applied(epoch, position)) +} + +/// Mark the txn of every stored row stored in its vShard's buffer, and +/// remove the rows of finished txns. Boot runs this once the applied +/// ledgers are filled, before the data-group apply loop and the schedulers +/// start. +pub fn restore_at_boot(shared: &SharedState) -> crate::Result<()> { + let catalog = shared.credentials.catalog(); + let mut finished = Vec::new(); + let mut waiting: BTreeMap> = BTreeMap::new(); + for row in catalog.load_calvin_barrier_logs(None)? { + if is_applied(shared, row.vshard_id, row.epoch, row.position) { + finished.push((row.vshard_id, row.epoch, row.position)); + } else { + waiting + .entry(row.vshard_id) + .or_default() + .push(TxnId::new(row.epoch, row.position)); + } + } + catalog.remove_calvin_barrier_logs(&finished)?; + for (vshard_id, txns) in waiting { + shared + .calvin + .read_results + .vshard(vshard_id) + .mark_stored(txns); + } + Ok(()) +} + +/// The stored rows of the unfinished txns of `vshards`, for a snapshot of +/// their data group. The caller fences the group's apply, so the rows hold +/// exactly the barrier entries at or below the snapshot's index. +/// +/// Fails when a vShard holds an event its row lacks: the rows do not hold +/// every entry the snapshot covers. The build retries on the next +/// heartbeat. +pub fn capture_group( + shared: &SharedState, + vshards: &BTreeSet, +) -> crate::Result> { + if let Some(vshard_id) = vshards + .iter() + .copied() + .find(|vshard_id| shared.calvin.read_results.vshard(*vshard_id).has_unsaved()) + { + return Err(crate::Error::Internal { + detail: format!( + "snapshot build: vShard {vshard_id} holds a dependent-read barrier event its \ + stored row lacks; the build retries on the next heartbeat" + ), + }); + } + let rows = shared + .credentials + .catalog() + .load_calvin_barrier_logs(Some(vshards))?; + Ok(rows + .into_iter() + .filter(|row| !is_applied(shared, row.vshard_id, row.epoch, row.position)) + .map(|row| CutBarrierLog { + vshard_id: row.vshard_id, + epoch: row.epoch, + position: row.position, + log: row.log, + }) + .collect()) +} + +/// Install `logs`, the rows a snapshot of the data group of `vshards` +/// carries: they replace this node's rows of `vshards`, and each vShard's +/// buffer holds their txns as stored. +pub fn install_group( + shared: &SharedState, + vshards: &BTreeSet, + logs: Vec, +) -> crate::Result<()> { + let rows: Vec = logs + .into_iter() + .filter(|log| vshards.contains(&log.vshard_id)) + .map(|log| StoredBarrierLog { + vshard_id: log.vshard_id, + epoch: log.epoch, + position: log.position, + log: log.log, + }) + .collect(); + shared + .credentials + .catalog() + .replace_calvin_barrier_logs(vshards, &rows)?; + for vshard_id in vshards { + let stored: BTreeSet = rows + .iter() + .filter(|row| row.vshard_id == *vshard_id) + .map(|row| TxnId::new(row.epoch, row.position)) + .collect(); + shared + .calvin + .read_results + .vshard(*vshard_id) + .reset_to_stored(stored); + } + Ok(()) +} + +/// Remove every row of `vshard_id`, which left this node, and its buffer. +pub fn forget_vshard(shared: &SharedState, vshard_id: u32) -> crate::Result<()> { + shared.calvin.read_results.forget(vshard_id); + shared + .credentials + .catalog() + .replace_calvin_barrier_logs(&BTreeSet::from([vshard_id]), &[]) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use nodedb_physical::physical_plan::meta::PassiveReadKeyId; + use nodedb_types::{QualifiedCollection, Value}; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::control::cluster::calvin::scheduler::NOT_YET_APPLIED_EPOCH; + use crate::wal::WalManager; + + const VSHARD: u32 = 5; + + fn state() -> (Arc, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + ( + SharedState::new(dispatcher, wal).expect("shared state"), + dir, + ) + } + + fn read(passive_vshard: u32, value: i64) -> BarrierEvent { + BarrierEvent::Read { + passive_vshard, + values: vec![( + PassiveReadKeyId::kv( + QualifiedCollection::from_stored("items".to_owned()), + b"k".to_vec(), + ), + Value::Integer(value), + )], + } + } + + /// Saved events fold in order into the row, and the row reloads. + #[test] + fn saved_events_fold_into_the_row() { + let (state, _dir) = state(); + let catalog = state.credentials.catalog(); + let txn = TxnId::new(4, 1); + save_event(catalog, VSHARD, txn, &read(1, 10)).expect("save"); + save_event(catalog, VSHARD, txn, &BarrierEvent::Timeout).expect("save"); + save_event(catalog, VSHARD, txn, &read(2, 20)).expect("save"); + let mut expected = BarrierLog::default(); + expected.note(read(1, 10)); + expected.note(BarrierEvent::Timeout); + assert_eq!(load_log(catalog, VSHARD, txn).expect("load"), expected); + remove_log(catalog, VSHARD, txn).expect("remove"); + assert_eq!( + load_log(catalog, VSHARD, txn).expect("load"), + BarrierLog::default() + ); + } + + /// Boot rebuilds the buffer from the stored rows: an unfinished txn's + /// events wait as stored, and a finished txn's row is removed. + #[test] + fn boot_rebuilds_the_buffer_from_the_rows() { + let (state, _dir) = state(); + let catalog = state.credentials.catalog(); + let open = TxnId::new(7, 0); + let done = TxnId::new(6, 0); + save_event(catalog, VSHARD, open, &read(1, 10)).expect("save"); + save_event(catalog, VSHARD, done, &read(1, 11)).expect("save"); + state + .calvin + .applied + .install(VSHARD, NOT_YET_APPLIED_EPOCH, BTreeSet::from([(6, 0)])); + + restore_at_boot(&state).expect("restore"); + + let buffer = state.calvin.read_results.vshard(VSHARD); + assert_eq!(buffer.waiting_txns(), 1); + let events = buffer.take(open).expect("the open txn waits"); + assert!(events.stored); + let mut expected = BarrierLog::default(); + expected.note(read(1, 10)); + assert_eq!(load_log(catalog, VSHARD, open).expect("load"), expected); + assert!( + catalog + .load_calvin_barrier_log(VSHARD, done.epoch, done.position) + .expect("load") + .is_none(), + "the finished txn's row is removed" + ); + } + + /// A snapshot carries the rows of unfinished txns, and its install + /// replaces the target's rows and buffer with them. + #[test] + fn a_snapshot_carries_the_rows_of_unfinished_txns() { + let (builder, _builder_dir) = state(); + let open = TxnId::new(3, 2); + save_event(builder.credentials.catalog(), VSHARD, open, &read(1, 10)).expect("save"); + save_event( + builder.credentials.catalog(), + VSHARD, + TxnId::new(2, 0), + &read(1, 9), + ) + .expect("save"); + builder + .calvin + .applied + .install(VSHARD, NOT_YET_APPLIED_EPOCH, BTreeSet::from([(2, 0)])); + let vshards = BTreeSet::from([VSHARD]); + let logs = capture_group(&builder, &vshards).expect("capture"); + assert_eq!(logs.len(), 1); + assert_eq!((logs[0].epoch, logs[0].position), (3, 2)); + + let (target, _target_dir) = state(); + let stale = TxnId::new(1, 0); + save_event(target.credentials.catalog(), VSHARD, stale, &read(1, 1)).expect("save"); + target + .calvin + .read_results + .vshard(VSHARD) + .push(stale, read(1, 1), 8, false); + install_group(&target, &vshards, logs).expect("install"); + + let buffer = target.calvin.read_results.vshard(VSHARD); + assert!(!buffer.waits(stale)); + assert!(!buffer.has_unsaved()); + assert!(buffer.take(open).expect("installed txn").stored); + let mut expected = BarrierLog::default(); + expected.note(read(1, 10)); + assert_eq!( + load_log(target.credentials.catalog(), VSHARD, open).expect("load"), + expected + ); + assert_eq!( + load_log(target.credentials.catalog(), VSHARD, stale).expect("load"), + BarrierLog::default() + ); + } + + /// A leftover row of a finished txn goes on the sweep, with no restart. + /// The row of an unfinished txn stays. + #[test] + fn the_sweep_removes_a_finished_txns_leftover_row() { + let (state, _dir) = state(); + let catalog = state.credentials.catalog(); + let done = TxnId::new(8, 0); + let open = TxnId::new(9, 0); + save_event(catalog, VSHARD, done, &read(1, 1)).expect("save"); + save_event(catalog, VSHARD, open, &read(1, 2)).expect("save"); + save_event(catalog, VSHARD + 1, done, &read(1, 3)).expect("save"); + let ledger = state.calvin.applied.get_or_create(VSHARD); + ledger.mark_terminal(done.epoch, done.position); + + assert_eq!(sweep_finished(catalog, VSHARD, &ledger).expect("sweep"), 1); + + let left: Vec<(u32, u64, u32)> = catalog + .load_calvin_barrier_logs(None) + .expect("load") + .into_iter() + .map(|row| (row.vshard_id, row.epoch, row.position)) + .collect(); + assert_eq!(left, vec![(VSHARD, 9, 0), (VSHARD + 1, 8, 0)]); + assert_eq!(sweep_finished(catalog, VSHARD, &ledger).expect("sweep"), 0); + } + + /// A vShard holding an unsaved event refuses the capture. + #[test] + fn an_unsaved_event_refuses_the_capture() { + let (state, _dir) = state(); + state + .calvin + .read_results + .vshard(VSHARD) + .push(TxnId::new(1, 0), read(1, 1), 8, false); + assert!(capture_group(&state, &BTreeSet::from([VSHARD])).is_err()); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/barrier.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/barrier.rs index ec202df1c..708ee73ef 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/barrier.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/barrier.rs @@ -1,63 +1,42 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Dependent-read barrier for Calvin scheduler. +//! Dependent-read barrier of the Calvin scheduler. //! -//! When a dependent-read Calvin txn arrives at an active vshard (one with -//! write keys), the scheduler inserts a [`PendingDependentBarrier`] and waits -//! for all passive vshards to deliver their read results via the per-vshard -//! Raft log (`ReplicatedWrite::CalvinReadResult`). +//! A dependent-read txn names the rows its passive vShards read. On an +//! active vShard (one the write set names) the data-group leader opens a +//! [`PendingDependentBarrier`] once the txn holds its locks. The barrier +//! waits for the read result of every passive vShard through the vShard's +//! data-group log (`ReplicatedWrite::CalvinReadResult`). //! -//! Once all passive vshards have delivered, the barrier assembles the -//! `injected_reads` map and dispatches `MetaOp::CalvinExecuteActive`. +//! The log alone decides the barrier, so every replica decides it alike. +//! The scheduler folds the log's barrier entries into a +//! [`crate::control::cluster::calvin::scheduler::BarrierLog`] per txn: //! -//! # Determinism +//! - Every result came first: the barrier completes. The leader compares the +//! values with the ones the coordinator read, and dispatches +//! `MetaOp::CalvinExecuteActive` or votes `PredictionDrift`. +//! - A `ReplicatedWrite::CalvinReadTimeout` came first: the barrier times +//! out, and the leader votes abort. //! -//! All maps here use `BTreeMap`/`BTreeSet` — never `HashMap`/`HashSet`. +//! The leader proposes the timeout entry once its barrier waited past +//! `tuning.calvin.dependent_read_passive_timeout_ms` on its own clock. The +//! clock only decides when the entry is proposed. Its place in the log +//! decides what it does. //! -//! # Timing +//! # Determinism //! -//! `Instant::now()` is used for `timeout_at` (observability / off-WAL path -//! only; barrier timeouts are never written to the WAL). +//! All maps here use `BTreeMap`/`BTreeSet` — never `HashMap`/`HashSet`. use std::collections::{BTreeMap, BTreeSet}; use std::time::Instant; -use nodedb_cluster::calvin::types::SequencedTxn; - -use super::super::lock_manager::TxnId; -use crate::types::TenantId; +use nodedb_cluster::calvin::types::{DependentReadSpec, SequencedTxn}; use nodedb_physical::physical_plan::meta::PassiveReadKeyId; use nodedb_types::Value; -// ── ReadResultEvent ──────────────────────────────────────────────────────────── - -/// A committed `CalvinReadResult` Raft entry delivered to the active vshard's -/// scheduler. -/// -/// Produced by the apply loop when a `ReplicatedWrite::CalvinReadResult` -/// entry commits on a vshard whose scheduler has an active dependent barrier. -#[derive(Debug)] -pub struct ReadResultEvent { - /// Sequencer epoch of the txn. - pub epoch: u64, - /// Position within the epoch batch. - pub position: u32, - /// Vshard that performed the read. - pub passive_vshard: u32, - /// Tenant scope. - pub tenant_id: TenantId, - /// Decoded read values from the passive participant. - pub values: Vec<(PassiveReadKeyId, Value)>, -} - -/// Read values received from passive vShards, keyed by passive vShard. -/// `BTreeMap` for determinism. -pub type ReceivedReads = BTreeMap>; - -// ── PendingDependentBarrier ──────────────────────────────────────────────────── +use super::super::lock_manager::TxnId; -/// A dependent-read barrier: tracks which passive vshards have delivered -/// their read results and assembles the `injected_reads` map once all are in. +/// A dependent-read barrier the data-group leader holds open. pub struct PendingDependentBarrier { /// Original sequenced transaction. pub txn: SequencedTxn, @@ -65,41 +44,86 @@ pub struct PendingDependentBarrier { /// reservation owns the lock). Carried through the barrier so the eventual /// `dispatch_active_txn` call can park the correct id in `PendingTxn`. pub lock_owner: TxnId, - /// Passive vshards still waiting to deliver their read results. - /// Entries are removed as `ReadResultEvent`s arrive. - /// `BTreeSet` for determinism. - pub waiting_for: BTreeSet, - /// Received read values, keyed by passive vshard. - pub received: ReceivedReads, - /// Deadline for passive participant delivery. - /// - /// `Instant::now()` used for barrier timeout (observability / off-WAL path). - pub timeout_at: Instant, + /// The passive vShards whose read result the barrier waits for. + pub passive: BTreeSet, + /// When this leader proposes the barrier's timeout entry, on its own + /// clock. A proposal re-arms it, so a timeout entry a leader change lost + /// is proposed again. + pub timeout_due: Instant, } -impl PendingDependentBarrier { - /// Whether all passive participants have delivered. - pub fn is_complete(&self) -> bool { - self.waiting_for.is_empty() +/// The first passive row whose broadcast value differs from the value the +/// coordinator read, described. `None` when every row matches. +/// +/// A row reads back as its stored bytes, `Value::Bytes`, or `Value::Null` +/// when absent. A row no passive vShard returned, or one returned as any +/// other value, differs. +pub fn expected_reads_drift( + spec: &DependentReadSpec, + injected: &BTreeMap, +) -> Option { + spec.expected.iter().find_map(|(row, expected)| { + let found = match injected.get(row) { + Some(Value::Bytes(bytes)) => Some(bytes.as_slice()), + Some(Value::Null) => None, + Some(other) => { + return Some(format!( + "the passive read of {row:?} returned {other:?}, not stored bytes" + )); + } + None => return Some(format!("no passive vShard returned {row:?}")), + }; + (found != expected.as_deref()) + .then(|| format!("{row:?} changed since the coordinator read it")) + }) +} + +#[cfg(test)] +mod tests { + use nodedb_types::QualifiedCollection; + + use super::*; + + fn row() -> PassiveReadKeyId { + PassiveReadKeyId::kv( + QualifiedCollection::from_stored("items".to_owned()), + b"alice:sword".to_vec(), + ) } - /// Assemble the full `injected_reads` map from all received values. - /// - /// `BTreeMap` ensures deterministic iteration order across replicas. - pub fn assemble_injected_reads(&self) -> BTreeMap { - let mut out = BTreeMap::new(); - for values in self.received.values() { - for (key_id, value) in values { - out.insert(key_id.clone(), value.clone()); - } + fn spec(expected: Option<&[u8]>) -> DependentReadSpec { + DependentReadSpec { + passive_reads: BTreeMap::new(), + expected: BTreeMap::from([(row(), expected.map(<[u8]>::to_vec))]), } - out } - /// Whether this barrier has timed out. - /// - /// `Instant::now()` used for timeout check (observability; off-WAL path). - pub fn is_timed_out(&self) -> bool { - Instant::now() > self.timeout_at // no-determinism: scheduler barrier timeout check, not Calvin WAL data + #[test] + fn matching_bytes_report_no_drift() { + let injected = BTreeMap::from([(row(), Value::Bytes(b"v1".to_vec()))]); + assert_eq!(expected_reads_drift(&spec(Some(b"v1")), &injected), None); + } + + #[test] + fn changed_bytes_report_drift() { + let injected = BTreeMap::from([(row(), Value::Bytes(b"v2".to_vec()))]); + assert!(expected_reads_drift(&spec(Some(b"v1")), &injected).is_some()); + } + + #[test] + fn a_row_gone_since_the_read_reports_drift() { + let injected = BTreeMap::from([(row(), Value::Null)]); + assert!(expected_reads_drift(&spec(Some(b"v1")), &injected).is_some()); + } + + #[test] + fn an_absent_row_still_absent_reports_no_drift() { + let injected = BTreeMap::from([(row(), Value::Null)]); + assert_eq!(expected_reads_drift(&spec(None), &injected), None); + } + + #[test] + fn a_row_no_passive_returned_reports_drift() { + assert!(expected_reads_drift(&spec(Some(b"v1")), &BTreeMap::new()).is_some()); } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/config.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/config.rs index 85ede01c4..89178f470 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/config.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/config.rs @@ -23,12 +23,13 @@ pub struct SchedulerConfig { /// Epoch duration in milliseconds (used to compute deadlines). Equal to /// the sequencer's epoch duration. pub epoch_duration_ms: u64, - /// Timeout for passive participant responses in dependent-read txns. + /// Wait of a dependent-read barrier for its passive read results before + /// the data-group leader proposes the barrier's timeout entry. /// - /// Default: 60 milliseconds, three default epochs. + /// Default: 5000 milliseconds, 250 default epochs. /// - /// `Instant::now()` is used for barrier timeouts (observability / off-WAL - /// path only; never influences WAL bytes). + /// The leader's clock decides only when it proposes the entry. The + /// entry's place in the log decides the barrier. pub dependent_read_passive_timeout_ms: u64, /// Stall deadline for a staged txn parked awaiting the durable global /// verdict. On expiry the scheduler re-probes the registry and, if the @@ -125,7 +126,7 @@ mod tests { assert_eq!(config.channel_capacity, 512); assert_eq!(config.txn_deadline_multiplier, 3); assert_eq!(config.epoch_duration_ms, 20); - assert_eq!(config.dependent_read_passive_timeout_ms, 60); + assert_eq!(config.dependent_read_passive_timeout_ms, 5_000); assert_eq!(config.verdict_stall_warn_ms, 5_000); assert_eq!(config.catch_up_window, 512); assert_eq!(config.restage_attempts, 5); diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs index ebe401cac..ebfb477c5 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/commit_resolve/verdict.rs @@ -47,6 +47,16 @@ impl Scheduler { txn_id: TxnId, committed: bool, ) { + // An abort verdict ends a txn still waiting at its barrier: it + // stages nothing, and its locks free at once. + if !committed && self.dependent_barrier.contains_key(&txn_id) { + self.abort_open_barrier( + txn_id, + nodedb_cluster::calvin::AbortReason::ParticipantError, + "an abort verdict reached the txn's open dependent-read barrier".to_owned(), + ); + return; + } let Some((state, writes)) = self .pending .get(&txn_id) @@ -62,6 +72,7 @@ impl Scheduler { } CommitState::AwaitingVerdict => self.resume_parked(txn_id, committed, writes), CommitState::Staged + | CommitState::ReadingPassive | CommitState::AwaitingResolveTurn | CommitState::AwaitingRedoResolve | CommitState::AwaitingRedoApply { .. } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs index dd1a0d0c2..8e13595db 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/completion_route.rs @@ -83,6 +83,9 @@ impl Scheduler { CommitState::AwaitingDrop => { self.finish_drop(txn_id, &response); } + CommitState::ReadingPassive => { + self.finish_passive_read(txn_id, &response); + } CommitState::Following | CommitState::AwaitingVerdict | CommitState::AwaitingResolveTurn diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs index 959df7b06..fe7766f9b 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/deferred.rs @@ -26,6 +26,8 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) enum DispatchSt StageStatic, /// `CalvinExecuteActive` stage of a dependent-read txn. StageActive, + /// `CalvinExecutePassive` read of a dependent-read txn. + ReadPassive, /// `CalvinResolve` of a committed staged txn. Resolve, /// `CalvinDrop` of a staged txn that ends with no log entry. The txn @@ -41,7 +43,11 @@ impl DispatchStep { /// Whether the txn waits for this step's answer. fn is_awaited(self) -> bool { match self { - Self::StageStatic | Self::StageActive | Self::Resolve | Self::Drop => true, + Self::StageStatic + | Self::StageActive + | Self::ReadPassive + | Self::Resolve + | Self::Drop => true, Self::Discard => false, } } @@ -214,6 +220,7 @@ impl Scheduler { ) { let halt_step = match step { DispatchStep::StageStatic | DispatchStep::StageActive => HaltStep::Stage, + DispatchStep::ReadPassive => HaltStep::PassiveRead, DispatchStep::Resolve => HaltStep::Resolve, DispatchStep::Drop | DispatchStep::Discard => HaltStep::Drop, }; @@ -280,7 +287,7 @@ impl Scheduler { } self.spawn_response_bridge(txn_id, request_id, resp_rx); } - DispatchStep::Resolve | DispatchStep::Drop => { + DispatchStep::ReadPassive | DispatchStep::Resolve | DispatchStep::Drop => { self.spawn_response_bridge(txn_id, request_id, resp_rx); } DispatchStep::Discard => { diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs index 08405f6cd..7e1ad7cf8 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/active_dispatch.rs @@ -51,12 +51,11 @@ impl Scheduler { match self.local_calvin_plans(plans, txn.tx_class.database_id, epoch, position) { Ok(p) if !p.is_empty() => p, Ok(_) => { - // A dependent-read active txn dispatched here always carries a - // local write slice (the OLLP orchestrator only routes the write - // participant through this path). An empty local slice is a - // routing bug, not a read-only participant, so it rejects the - // plans rather than dispatching an active task with nothing - // to apply. + // Only a vShard the write set names opens a barrier, so an + // active txn dispatched here carries a local write slice. An + // empty local slice means the plans and the write set + // disagree, so it rejects the plans rather than dispatching + // an active task with nothing to apply. let e = crate::Error::Internal { detail: format!( "calvin active txn {epoch}/{position} homes no local write plans \ diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/follow.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/follow.rs index 83a3790b9..80094c351 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/follow.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/dispatch/follow.rs @@ -5,7 +5,8 @@ //! The follower holds the txn's locks and stages nothing. It owes no vote. //! The txn completes on an abort verdict, on a COMMIT verdict for a slice //! with no local write, or once the slice's stamped redo applies from the -//! log. A promotion stages it. +//! log. A promotion stages it. A dependent-read txn keeps its barrier log +//! meanwhile. use std::time::Instant; @@ -25,7 +26,8 @@ impl Scheduler { txn_id: TxnId, lock_owner: TxnId, ) { - let scope = self.follow_scope(&txn, txn_id); + let scope = self.local_slice_scope(&txn, txn_id); + let dependent = txn.tx_class.dependent_reads.is_some(); self.pending.insert( txn_id, PendingTxn { @@ -47,6 +49,11 @@ impl Scheduler { ungated: false, }, ); + // A dependent txn keeps its barrier log from its grant on, so a + // promotion decides its barrier from every entry the log holds. + if dependent { + self.absorb_buffered_reads(txn_id); + } if let Some(verdict) = self.registry.verdict(nodedb_cluster::calvin::TxnId::new( txn_id.epoch, txn_id.position, @@ -55,9 +62,16 @@ impl Scheduler { } } - /// The scope of `txn`'s slice on this vShard. Plans that do not decode - /// or route make the leader vote abort, so they hold no write here. - fn follow_scope(&self, txn: &SequencedTxn, txn_id: TxnId) -> SliceScope { + /// The scope of `txn`'s slice on this vShard, from the txn's own plans. + /// A follower completes a COMMIT with no entry only when this scope + /// writes nothing, so it never comes from a leader's step in progress. + /// Plans that do not decode or route make the leader vote abort, so they + /// hold no write here. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn local_slice_scope( + &self, + txn: &SequencedTxn, + txn_id: TxnId, + ) -> SliceScope { let local = super::super::super::helpers::decode_plans(&txn.tx_class.plans).and_then(|plans| { self.local_calvin_plans( diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs index 435dfd721..47f3dfadd 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/halt.rs @@ -118,6 +118,8 @@ pub(in crate::control::cluster::calvin::scheduler::driver::core) enum HaltStep { IdentityBind, /// The wait for this node's metadata apply to reach the txn's floor. MetadataHold, + /// The passive read of a dependent-read txn. + PassiveRead, } impl HaltStep { @@ -133,6 +135,7 @@ impl HaltStep { Self::Drop => "drop", Self::IdentityBind => "identity_bind", Self::MetadataHold => "metadata_hold", + Self::PassiveRead => "passive_read", } } @@ -147,6 +150,7 @@ impl HaltStep { CommitState::AwaitingRedoResolve | CommitState::AwaitingResolveTurn => Self::Resolve, CommitState::AwaitingRedoApply { .. } | CommitState::Following => Self::RedoApply, CommitState::AwaitingDrop => Self::Drop, + CommitState::ReadingPassive => Self::PassiveRead, } } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs index a77330fe6..5583080ca 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/mod.rs @@ -13,9 +13,10 @@ //! //! All bookkeeping uses `BTreeMap`/`BTreeSet` — never `HashMap`/`HashSet` — //! and dispatch order is `(epoch, position)` order. `Instant::now()` is used -//! only for lock-wait latency metrics, the dependent-read barrier -//! `timeout_at`, and the restage backoff, all off the WAL-influencing path; -//! every call site carries a `// no-determinism:` marker. +//! only for lock-wait latency metrics, the instant a leader proposes a +//! dependent-read barrier's timeout entry, and the restage backoff, all off +//! the WAL-influencing path; every call site carries a `// no-determinism:` +//! marker. mod apply_result; pub mod catch_up; @@ -34,9 +35,11 @@ pub mod owed; pub mod parts; mod parts_error; pub mod parts_lane; +mod passive_read; pub mod process; pub mod propose; pub mod read_result; +mod read_timeout; pub mod redo_applied; pub mod redo_propose; pub mod request; diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/passive_read.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/passive_read.rs new file mode 100644 index 000000000..74368bb1b --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/passive_read.rs @@ -0,0 +1,491 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A dependent-read txn on the data-group leader of one of its vShards. +//! +//! The vShard's role in the txn decides the route: +//! +//! - passive (the txn reads rows here): the leader dispatches +//! `MetaOp::CalvinExecutePassive` under the txn's locks, and proposes the +//! values it returns to the data group of every active vShard; +//! - active (the write set names this vShard): the leader opens the +//! dependent-read barrier, once its own passive read answered when the +//! vShard is passive too; +//! - neither: a read-set participant, staged as a static slice. +//! +//! A passive-only vShard writes nothing. Its read answer is its vote: a +//! successful read votes commit, a failed one votes abort. + +use std::sync::Arc; +use std::time::Instant; + +use nodedb_cluster::calvin::AbortReason; +use nodedb_cluster::calvin::types::SequencedTxn; +use nodedb_physical::physical_plan::PhysicalPlan; +use nodedb_physical::physical_plan::meta::MetaOp; + +use super::deferred::{DispatchOutcome, DispatchStep}; +use super::halt::error_response_text; +use super::propose::{CalvinReadResultProposal, propose_calvin_read_result}; +use super::scheduler::Scheduler; +use crate::bridge::envelope::{Response, Status}; +use crate::control::cluster::calvin::scheduler::driver::types::{CommitState, PendingTxn}; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; + +/// This vShard's role in a dependent-read txn. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct DependentRole { + /// The txn reads rows on this vShard. + passive: bool, + /// The txn's write set names this vShard. + active: bool, +} + +impl Scheduler { + /// Route a granted dependent-read txn on this leader by the vShard's + /// role in it. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn route_dependent( + &mut self, + txn: SequencedTxn, + txn_id: TxnId, + lock_owner: TxnId, + ) { + let role = match self.dependent_role(&txn) { + Ok(role) => role, + Err(error) => { + self.reject_plan(txn, txn_id, lock_owner, &error); + return; + } + }; + match role { + DependentRole { passive: true, .. } => { + self.dispatch_passive_read(txn, txn_id, lock_owner); + } + DependentRole { + passive: false, + active: true, + } => self.insert_dependent_barrier(txn, txn_id, lock_owner), + DependentRole { + passive: false, + active: false, + } => self.dispatch_txn(txn, txn_id, lock_owner), + } + } + + /// This vShard's role in dependent-read txn `txn`. + fn dependent_role(&self, txn: &SequencedTxn) -> crate::Result { + let passive = txn + .tx_class + .dependent_reads + .as_ref() + .is_some_and(|spec| spec.passive_reads.contains_key(&self.vshard_id)); + let active = txn + .tx_class + .active_vshards() + .map_err(|error| crate::Error::Internal { + detail: format!( + "calvin txn {}/{}: the write set names no vShard of its database: {error}", + txn.epoch, txn.position + ), + })? + .contains(&self.vshard_id); + Ok(DependentRole { passive, active }) + } + + /// Dispatch the passive read of `txn` on this vShard, and hold the txn + /// in [`CommitState::ReadingPassive`] until it answers. + fn dispatch_passive_read(&mut self, txn: SequencedTxn, txn_id: TxnId, lock_owner: TxnId) { + let keys_to_read = txn + .tx_class + .dependent_reads + .as_ref() + .and_then(|spec| spec.passive_reads.get(&self.vshard_id)) + .cloned() + .unwrap_or_default(); + let tenant_id = txn.tx_class.tenant_id; + let database_id = txn.tx_class.database_id; + let plan = PhysicalPlan::Meta(MetaOp::CalvinExecutePassive { + epoch: txn_id.epoch, + position: txn_id.position, + tenant_id, + keys_to_read, + }); + let request_id = self.next_request_id(); + let request = self.build_exempt_request(request_id, tenant_id, database_id, plan); + // A passive-only vShard votes on its read, so it checks the + // collections the txn names as a stage does. + let (superseded, gates) = self.check_incarnations(&txn.tx_class); + // The slice's own scope: a vShard that is passive and active writes + // here, whatever step the txn is at. + let scope = self.local_slice_scope(&txn, txn_id); + self.pending.insert( + txn_id, + PendingTxn { + txn, + lock_owner, + // no-determinism: dispatch time is observability only. + dispatch_time: Instant::now(), + has_primary_write: false, + raises_write_mark: false, + has_returning: false, + commit_state: CommitState::ReadingPassive, + // The dispatch below records its request. + awaiting: None, + verdict_deadline: None, + stage_error: None, + scope, + redo: None, + superseded, + gates, + ungated: false, + }, + ); + self.absorb_buffered_reads(txn_id); + if let DispatchOutcome::Failed(error) = + self.dispatch_sequenced(txn_id, DispatchStep::ReadPassive, request) + { + self.fail_dispatch_step(txn_id, DispatchStep::ReadPassive, error); + } + } + + /// Act on the answer of `txn_id`'s passive read: propose the values to + /// every active vShard, then vote on them on a passive-only vShard, or + /// open the barrier on an active one. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn finish_passive_read( + &mut self, + txn_id: TxnId, + response: &Response, + ) { + let Some(role) = self + .pending + .get(&txn_id) + .map(|pending| self.dependent_role(&pending.txn)) + else { + return; + }; + let role = match role { + Ok(role) => role, + Err(error) => { + if let Some(pending) = self.pending.remove(&txn_id) { + self.reject_plan(pending.txn, txn_id, pending.lock_owner, &error); + } + return; + } + }; + let read = response.status == Status::Ok; + if read { + self.propose_read_results(txn_id, response.payload.to_vec()); + } + if !role.active { + // A passive-only vShard writes nothing: its read answer carries + // its vote, and a COMMIT verdict drops the empty slice. + self.resolve_staged_commit(txn_id, response); + return; + } + let Some(pending) = self.pending.remove(&txn_id) else { + return; + }; + if read { + self.insert_dependent_barrier(pending.txn, txn_id, pending.lock_owner); + } else { + self.park_unstaged( + pending.txn, + txn_id, + pending.lock_owner, + AbortReason::ParticipantError, + error_response_text("passive read", response), + ); + } + } + + /// Propose `values`, this vShard's passive read of `txn_id`, to the data + /// group of every active vShard of the txn. Each proposal runs on its + /// own task until `config.passive_timeout()` passes. + fn propose_read_results(&self, txn_id: TxnId, values: Vec) { + let Some(pending) = self.pending.get(&txn_id) else { + return; + }; + let tx_class = &pending.txn.tx_class; + let targets = match tx_class.active_vshards() { + Ok(targets) => targets, + Err(error) => { + tracing::error!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + %error, + "calvin: no active vShard to propose the passive read to" + ); + return; + } + }; + #[cfg(feature = "failpoints")] + let gates: Vec = tx_class + .dependent_reads + .as_ref() + .and_then(|spec| spec.passive_reads.get(&self.vshard_id)) + .map(|keys| { + keys.iter() + .map(|key| { + format!( + "calvin::before_read_result_propose::{}", + key.engine_key.collection() + ) + }) + .collect() + }) + .unwrap_or_default(); + let budget = self.config.passive_timeout(); + for target_vshard in targets { + let proposal = CalvinReadResultProposal { + target_vshard, + epoch: txn_id.epoch, + position: txn_id.position, + tenant_id: tx_class.tenant_id, + database_id: tx_class.database_id, + passive_vshard: self.vshard_id, + values: values.clone(), + }; + let state = Arc::downgrade(&self.shared); + #[cfg(feature = "failpoints")] + let gates = gates.clone(); + tokio::spawn(async move { + #[cfg(feature = "failpoints")] + for gate in &gates { + crate::control::fail_gate::wait(gate).await; + } + let deadline = tokio::time::Instant::now() + budget; + if let Err(error) = propose_calvin_read_result(state, proposal, deadline).await { + tracing::warn!( + target_vshard, + epoch = txn_id.epoch, + position = txn_id.position, + %error, + "calvin: passive read result not proposed before its deadline; \ + the active barrier's timeout entry ends the txn" + ); + } + }); + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use nodedb_cluster::calvin::types::{ + DependentReadSpec, EngineKeySet, PassiveReadKey, SchedulerInput, SortedVec, + }; + use nodedb_cluster::calvin::{CalvinCompletionRegistry, SequencerEntry}; + + use super::*; + use crate::bridge::dispatch::CoreChannelDataSide; + use crate::bridge::envelope::{ErrorCode, Payload, StageVote}; + use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ + CapturingProposer, lead_data_group, + }; + use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ + build_test_scheduler_with_data_side, make_local_write_txn, test_coll_vshard, + }; + + /// A dependent txn that writes vShard `active` only, and reads `items` + /// on the scheduler's own vShard. + fn passive_only_txn(epoch: u64) -> SequencedTxn { + let mut txn = make_local_write_txn(epoch, 0); + let other = (0u32..4096) + .map(|i| format!("passive_target_{i}")) + .find(|name| { + nodedb_types::CollectionKey::from_bare(nodedb_types::DatabaseId::DEFAULT, name) + .vshard() + .as_u32() + != test_coll_vshard() + }) + .expect("a collection on another vShard"); + txn.tx_class.write_set = + nodedb_cluster::calvin::types::ReadWriteSet::new(vec![EngineKeySet::Document { + collection: other, + surrogates: SortedVec::new(vec![1]), + }]); + txn.tx_class.dependent_reads = Some(DependentReadSpec { + passive_reads: BTreeMap::from([( + test_coll_vshard(), + vec![PassiveReadKey { + engine_key: EngineKeySet::Kv { + collection: "test_coll".to_owned(), + keys: SortedVec::new(vec![b"k".to_vec()]), + }, + }], + )]), + expected: BTreeMap::new(), + }); + txn + } + + fn leader() -> ( + Scheduler, + tempfile::TempDir, + CoreChannelDataSide, + Arc, + ) { + let registry = CalvinCompletionRegistry::new_detached(); + let (mut scheduler, dir, data_side) = + build_test_scheduler_with_data_side(test_coll_vshard(), registry); + let proposer = CapturingProposer::accepting(); + scheduler.sequencer_proposer = proposer.clone(); + lead_data_group(&mut scheduler); + (scheduler, dir, data_side, proposer) + } + + /// The request id of the passive read the Data Plane received. + fn passive_read_request( + data_side: &mut CoreChannelDataSide, + ) -> Option { + let mut found = None; + while let Ok(request) = data_side.request_rx.try_pop() { + if matches!( + request.inner.plan, + PhysicalPlan::Meta(MetaOp::CalvinExecutePassive { .. }) + ) { + found = Some(request.inner.request_id); + } + } + found + } + + fn answer(request_id: crate::types::RequestId, status: Status) -> Response { + Response { + request_id, + status, + attempt: 1, + partial: false, + payload: Payload::from_vec( + zerompk::to_msgpack_vec(&Vec::<( + nodedb_physical::physical_plan::meta::PassiveReadKeyId, + nodedb_types::Value, + )>::new()) + .expect("encode"), + ), + watermark_lsn: crate::types::Lsn::ZERO, + error_code: (status == Status::Error).then(|| { + Box::new(ErrorCode::Internal { + detail: "read failed".into(), + }) + }), + stage_vote: (status == Status::Ok).then_some(StageVote::Commit), + read_versions: crate::types::ReadVersions::new(), + write_set: Vec::new(), + } + } + + /// A passive-only vShard dispatches its read under the txn's locks and + /// votes commit on a successful answer. + #[tokio::test] + async fn a_passive_only_vshard_reads_then_votes_commit() { + let (mut scheduler, _dir, mut data_side, proposer) = leader(); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(passive_only_txn(70)))); + let txn_id = TxnId::new(70, 0); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::ReadingPassive) + ); + let request_id = passive_read_request(&mut data_side).expect("the passive read"); + + scheduler.handle_completion(txn_id, request_id, Some(answer(request_id, Status::Ok))); + + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::AwaitingVerdict) + ); + assert!(proposer.accepted().iter().any(|entry| matches!( + entry, + SequencerEntry::Vote { + epoch: 70, + position: 0, + .. + } + ))); + } + + /// A failed passive read votes abort on a passive-only vShard. + #[tokio::test] + async fn a_failed_passive_read_votes_abort() { + let (mut scheduler, _dir, mut data_side, proposer) = leader(); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(passive_only_txn(71)))); + let txn_id = TxnId::new(71, 0); + let request_id = passive_read_request(&mut data_side).expect("the passive read"); + + scheduler.handle_completion(txn_id, request_id, Some(answer(request_id, Status::Error))); + + assert!(proposer.accepted().iter().any(|entry| matches!( + entry, + SequencerEntry::AbortVote { + epoch: 71, + reason: AbortReason::ParticipantError, + .. + } + ))); + } + + /// A vShard both passive and active opens its barrier once its own read + /// answered. The barrier waits for its own result through the log. + #[tokio::test] + async fn a_passive_and_active_vshard_opens_its_barrier_after_reading() { + let (mut scheduler, _dir, mut data_side, _proposer) = leader(); + let mut txn = make_local_write_txn(72, 0); + txn.tx_class.dependent_reads = Some(DependentReadSpec { + passive_reads: BTreeMap::from([(test_coll_vshard(), Vec::new())]), + expected: BTreeMap::new(), + }); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(txn))); + let txn_id = TxnId::new(72, 0); + let request_id = passive_read_request(&mut data_side).expect("the passive read"); + + scheduler.handle_completion(txn_id, request_id, Some(answer(request_id, Status::Ok))); + + assert!(!scheduler.pending.contains_key(&txn_id)); + assert!(scheduler.dependent_barrier.contains_key(&txn_id)); + } + + /// A leader that loses the vShard while its passive read runs follows + /// the txn with the slice's own write scope. A COMMIT verdict then waits + /// for the slice's redo, and the position stays unapplied until it + /// installs. With the passive read's empty scope, the follower ended the + /// COMMIT with no entry, marked the position applied, and skipped the + /// redo the next leader proposed. + #[tokio::test] + async fn a_demoted_passive_and_active_leader_waits_for_the_redo() { + let (mut scheduler, _dir, mut data_side, _proposer) = leader(); + let mut txn = make_local_write_txn(73, 0); + txn.tx_class.dependent_reads = Some(DependentReadSpec { + passive_reads: BTreeMap::from([(test_coll_vshard(), Vec::new())]), + expected: BTreeMap::new(), + }); + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(txn))); + let txn_id = TxnId::new(73, 0); + assert!(passive_read_request(&mut data_side).is_some()); + assert!( + scheduler + .pending + .get(&txn_id) + .is_some_and(|pending| pending.scope.writes), + "the passive read holds the slice's write scope" + ); + + scheduler.demote("the test moves the vShard's leadership"); + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Following) + ); + scheduler.resume_on_verdict(txn_id, true); + + assert_eq!( + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Following), + "a COMMIT for a write slice waits for its redo" + ); + assert!( + !scheduler.ledger.is_applied(73, 0), + "the position stays unapplied, so the redo installs" + ); + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs index 16fd6f776..7fd903831 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/process.rs @@ -3,7 +3,7 @@ //! New-txn processing, dependent-read barrier setup, and txn-completion //! bookkeeping for the Calvin scheduler. -use std::collections::BTreeMap; +use std::collections::BTreeSet; use std::sync::Arc; use std::time::Instant; @@ -257,7 +257,8 @@ impl Scheduler { } /// Route a granted, assembled txn: a follower holds it, the leader - /// stages it as a static dispatch or a dependent barrier. + /// stages it as a static dispatch, or routes a dependent-read txn by its + /// role on this vShard. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn route_granted( &mut self, txn: SequencedTxn, @@ -268,38 +269,45 @@ impl Scheduler { self.follow(txn, txn_id, lock_owner); return; } - let is_dependent = txn.tx_class.dependent_reads.is_some(); - if is_dependent { - self.insert_dependent_barrier(txn, txn_id, lock_owner); + if txn.tx_class.dependent_reads.is_some() { + self.route_dependent(txn, txn_id, lock_owner); } else { self.dispatch_txn(txn, txn_id, lock_owner); } } - /// Insert a dependent-read barrier for an active vshard. - fn insert_dependent_barrier(&mut self, txn: SequencedTxn, txn_id: TxnId, lock_owner: TxnId) { - let spec = match &txn.tx_class.dependent_reads { - Some(s) => s, + /// Open a dependent-read barrier for `txn` on this active vShard. + /// + /// The barrier waits for the read result of every passive vShard. This + /// leader proposes the barrier's timeout entry once it waited past + /// `config.passive_timeout()`. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn insert_dependent_barrier( + &mut self, + txn: SequencedTxn, + txn_id: TxnId, + lock_owner: TxnId, + ) { + let passive: BTreeSet = match &txn.tx_class.dependent_reads { + Some(spec) => spec.passive_vshards().collect(), None => { - // Shouldn't happen; fall through to static dispatch. - self.dispatch_txn(txn, txn_id, lock_owner); + let error = crate::Error::Internal { + detail: format!( + "calvin txn {}/{} reached a dependent-read barrier with no \ + dependent reads", + txn_id.epoch, txn_id.position + ), + }; + self.reject_plan(txn, txn_id, lock_owner, &error); return; } }; - - let waiting_for: std::collections::BTreeSet = - spec.passive_reads.keys().copied().collect(); - // no-determinism: passive barrier timeout is scheduler observability, not Calvin WAL data - let timeout_at = Instant::now() + self.config.passive_timeout(); - let barrier = PendingDependentBarrier { txn, lock_owner, - waiting_for, - received: BTreeMap::new(), - timeout_at, + passive, + // no-determinism: the leader's clock decides only when it proposes the timeout entry; the entry's place in the log decides the barrier. + timeout_due: Instant::now() + self.config.passive_timeout(), }; - self.open_barrier(txn_id, barrier); } @@ -347,6 +355,7 @@ impl Scheduler { mark: LedgerMark, ) { // The txn restages no more, and no barrier of it opens again. + let held_log = self.barrier_logs.contains_key(&txn_id); self.forget_held_state(txn_id); // Release this txn's locks. `release` promotes any waiter queued behind // each freed key to holder (moving it pending -> held) and returns the @@ -370,6 +379,10 @@ impl Scheduler { LedgerMark::Terminal => self.ledger.mark_terminal(txn_id.epoch, txn_id.position), LedgerMark::ByApply => {} } + // The ledger holds the position now, so the apply loop buffers and + // stores no later barrier entry of the txn. Each one it took before + // is spent. + self.forget_barrier_entries(txn_id, held_log); if let Some(watermark) = folded { self.publish_watermark(watermark); } @@ -456,6 +469,7 @@ impl Scheduler { #[cfg(test)] mod tests { use super::*; + use std::collections::BTreeMap; use std::sync::atomic::Ordering; use nodedb_cluster::calvin::CalvinCompletionRegistry; diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/propose.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/propose.rs index 58c329dc5..70d7fd42c 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/propose.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/propose.rs @@ -1,56 +1,125 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Propose `ReplicatedWrite::CalvinReadResult` Raft entries from passive -//! participants. +//! Propose a passive participant's read values to an active vShard's data +//! group as a `ReplicatedWrite::CalvinReadResult` entry. +//! +//! The passive vShard's data-group leader proposes to the data group of each +//! active vShard. That group's leader is often another node, so the entry +//! travels through the node's async propose phase, which forwards it to the +//! group leader. The phase takes no vShard admission slot: the entry writes +//! no row, and the target leader's write gate exempts it. +//! +//! A refused proposal is retried until the deadline. A second copy of an +//! entry that already landed changes nothing: the first result of a passive +//! vShard is the one a barrier counts. A proposal that never lands leaves +//! the active barrier waiting, and its leader's timeout entry ends it on +//! every replica alike. +use std::sync::Weak; +use std::time::Duration; + +use crate::control::state::SharedState; use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite}; use crate::types::{DatabaseId, TenantId}; -/// The passive read result to replicate: identity of the originating vshard -/// plus the sequencer coordinate, scope, and read values. +/// Pause between two attempts of a refused proposal. +const RETRY_PAUSE: Duration = Duration::from_millis(20); + +/// One passive vShard's read values, for one active vShard's data group. pub struct CalvinReadResultProposal { - pub vshard_id: u32, + /// The active vShard whose data group the entry goes to. + pub target_vshard: u32, pub epoch: u64, pub position: u32, pub tenant_id: TenantId, pub database_id: DatabaseId, + /// The vShard that read the values. pub passive_vshard: u32, - pub values_payload: Vec, + /// The zerompk-encoded `Vec<(PassiveReadKeyId, Value)>` the passive read + /// returned. + pub values: Vec, } -/// Propose a `ReplicatedWrite::CalvinReadResult` entry to the per-vshard -/// Raft group after a passive participant responds. -/// -/// This is a Control Plane function (Tokio async context). It constructs the -/// Raft entry and hands it to the WAL replication layer. +/// Propose `proposal` to the data group of `proposal.target_vshard`, and +/// wait for this node's apply of it when this node replicates that group. /// -/// `Instant::now()` is NOT used here (this goes to the WAL/Raft path). +/// Retries a refused proposal until `deadline`. Returns the last refusal +/// once the deadline passed. Returns `Ok` once an attempt landed in the +/// target leader's log, whatever this node's apply wait reports: a node that +/// replicates no copy of the group never applies it. pub async fn propose_calvin_read_result( - raft_proposer: &(dyn Fn(u32, Vec) -> crate::Result<(u64, u64)> + Send + Sync), + state: Weak, proposal: CalvinReadResultProposal, + deadline: tokio::time::Instant, ) -> crate::Result<()> { let CalvinReadResultProposal { - vshard_id, + target_vshard, epoch, position, tenant_id, database_id, passive_vshard, - values_payload, + values, } = proposal; let entry = ReplicatedEntry::new( tenant_id.as_u64(), database_id.as_u64(), - vshard_id, + target_vshard, ReplicatedWrite::CalvinReadResult { epoch, position, passive_vshard, tenant_id: tenant_id.as_u64(), - values: values_payload, + values, }, ); - let data = entry.encode()?; - raft_proposer(vshard_id, data)?; - Ok(()) + let bytes = entry.encode()?; + loop { + let submit = { + let Some(state) = state.upgrade() else { + return Err(crate::Error::Internal { + detail: "calvin read result not proposed: the node is shutting down".into(), + }); + }; + std::sync::Arc::clone(state.async_raft_submit()?) + }; + match submit( + target_vshard, + entry.idempotency_key, + bytes.clone(), + deadline, + ) + .await + { + Ok(proposed) => { + if let Err(error) = proposed.applied.await { + tracing::debug!( + target_vshard, + epoch, + position, + %error, + "calvin: read result proposed; this node's apply wait ended without it" + ); + } + return Ok(()); + } + Err(error) => { + if tokio::time::Instant::now() >= deadline { + return Err(error); + } + tracing::debug!( + target_vshard, + epoch, + position, + %error, + "calvin: read result proposal refused; proposing it again" + ); + tokio::time::sleep( + RETRY_PAUSE + .min(deadline.saturating_duration_since(tokio::time::Instant::now())), + ) + .await; + } + } + } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs index 054126a7f..53b0bc8bc 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_result.rs @@ -1,331 +1,665 @@ // SPDX-License-Identifier: BUSL-1.1 -//! `CalvinReadResult` handling and dependent-read barrier timeout sweeps. +//! Dependent-read barrier entries and the barrier's decision. //! -//! A read result arrives once, from the data-group log. A dependent txn this -//! scheduler holds with no barrier keeps the results it received: a -//! follower's held txn, or a leader's txn whose barrier timed out. A later -//! barrier of the txn, after a promotion or a restage, starts from them. +//! The data-group apply loop folds every `CalvinReadResult` and +//! `CalvinReadTimeout` entry into the txn's stored barrier row and this +//! vShard's read-result buffer. The scheduler takes a txn's entries into its +//! `barrier_logs` once it holds the txn: at the grant, and on each push for a +//! txn it holds. A txn the buffer holds as stored reads its row. A barrier then +//! decides from the log alone (see +//! [`crate::control::cluster::calvin::scheduler::BarrierLog`]), so every +//! replica that leads the vShard decides it alike: +//! +//! - complete: the leader checks the values against the ones the coordinator +//! read, and stages the active slice or votes `PredictionDrift`; +//! - timed out: the leader votes abort; +//! - waiting: the barrier stays open, unless an abort verdict is stored. +//! +//! A follower keeps the log of each dependent txn it holds. A promotion +//! opens the barrier from it and reaches the decision the earlier leader +//! reached. + +use std::sync::Arc; + +use tracing::{error, info, warn}; -use tracing::warn; +use nodedb_cluster::calvin::AbortReason; -use super::super::barrier::{PendingDependentBarrier, ReadResultEvent, ReceivedReads}; +use super::super::barrier::{PendingDependentBarrier, expected_reads_drift}; use super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::barrier_store; use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; +use crate::control::cluster::calvin::scheduler::{BarrierLog, BarrierOutcome, BufferedEvents}; impl Scheduler { - /// Handle a `ReadResultEvent` from the per-vshard Raft apply loop. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn handle_read_result( + /// Take the buffered barrier entries of every dependent txn this + /// scheduler holds, and decide each open barrier they reach. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn drain_read_results( &mut self, - event: ReadResultEvent, ) { - let txn_id = TxnId::new(event.epoch, event.position); - - let barrier = match self.dependent_barrier.get_mut(&txn_id) { - Some(b) => b, - None => { - if self.holds_dependent(txn_id) { - self.hold_reads( - txn_id, - ReceivedReads::from([(event.passive_vshard, event.values)]), - ); - return; - } - // Neither a barrier nor a held txn: the txn finished, or its - // input has not been granted here. Log and ignore. - warn!( - vshard_id = self.vshard_id, - epoch = event.epoch, - position = event.position, - passive_vshard = event.passive_vshard, - "calvin: received CalvinReadResult for unknown txn; ignoring" - ); - return; + let buffer = Arc::clone(&self.read_results); + let taken = buffer.take_held(|txn_id| self.holds_barrier_txn(txn_id)); + for (txn_id, events) in taken { + if self.absorb(txn_id, events) { + self.decide_barrier(txn_id); } - }; - - barrier.waiting_for.remove(&event.passive_vshard); - barrier.received.insert(event.passive_vshard, event.values); - - if !barrier.is_complete() { - return; } - - // All passive results in — remove barrier and dispatch active. - let Some(barrier) = self.dependent_barrier.remove(&txn_id) else { - return; - }; - self.dispatch_complete_barrier(txn_id, barrier); } - /// Open `barrier` for `txn_id`, starting from the reads the txn holds. - /// A barrier those reads complete dispatches at once. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn open_barrier( + /// Take the buffered barrier entries of `txn_id`, which this scheduler + /// holds from now on. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn absorb_buffered_reads( &mut self, txn_id: TxnId, - mut barrier: PendingDependentBarrier, ) { - if let Some(held) = self.held_reads.remove(&txn_id) { - for (passive_vshard, values) in held { - barrier.waiting_for.remove(&passive_vshard); - barrier.received.insert(passive_vshard, values); - } + if let Some(events) = self.read_results.take(txn_id) { + self.absorb(txn_id, events); } - if barrier.is_complete() { - self.dispatch_complete_barrier(txn_id, barrier); + } + + /// Fold `events` of `txn_id` into its barrier log. A stored txn's + /// events are its row's, extended with the ones memory held. Returns + /// whether the log took them. + /// + /// A row this node cannot read goes back to the buffer, still stored: + /// the barrier stays open, and the next pass reads the row again. + fn absorb(&mut self, txn_id: TxnId, events: BufferedEvents) -> bool { + let mut log = if events.stored { + match barrier_store::load_log(self.shared.credentials.catalog(), self.vshard_id, txn_id) + { + Ok(stored) => stored, + Err(e) => { + error!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + error = %e, + "calvin: the barrier row of a held txn does not load; reading it again \ + on the next pass" + ); + crate::diag::calvin_barrier_log_store_failed( + self.vshard_id, + Some((txn_id.epoch, txn_id.position)), + "load", + &e, + ); + self.read_results.restore(txn_id, events); + return false; + } + } } else { - self.dependent_barrier.insert(txn_id, barrier); - } + BarrierLog::default() + }; + log.extend(events.memory); + self.barrier_logs.entry(txn_id).or_default().extend(log); + true } - /// Dispatch the active stage of a barrier every passive vShard answered. - fn dispatch_complete_barrier(&mut self, txn_id: TxnId, barrier: PendingDependentBarrier) { - let injected_reads = barrier.assemble_injected_reads(); - self.dispatch_active_txn(barrier.txn, txn_id, barrier.lock_owner, injected_reads); + /// Whether `txn_id` is a dependent txn this scheduler holds: at a + /// barrier, or in `pending`. + fn holds_barrier_txn(&self, txn_id: TxnId) -> bool { + self.dependent_barrier.contains_key(&txn_id) + || self + .pending + .get(&txn_id) + .is_some_and(|pending| pending.txn.tx_class.dependent_reads.is_some()) } - /// Keep `reads` for `txn_id`, a dependent txn in `pending`. Reads for a - /// txn that left `pending` are dropped: it finished. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn hold_reads( + /// Open `barrier` for `txn_id` and decide it from the entries the log + /// holds already. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn open_barrier( &mut self, txn_id: TxnId, - reads: ReceivedReads, + barrier: PendingDependentBarrier, ) { - if reads.is_empty() || !self.holds_dependent(txn_id) { - return; - } - self.held_reads.entry(txn_id).or_default().extend(reads); - } - - /// Whether `txn_id` is a dependent txn in `pending`. - fn holds_dependent(&self, txn_id: TxnId) -> bool { - self.pending - .get(&txn_id) - .is_some_and(|pending| pending.txn.tx_class.dependent_reads.is_some()) + self.dependent_barrier.insert(txn_id, barrier); + self.absorb_buffered_reads(txn_id); + self.decide_barrier(txn_id); } - /// Check all pending dependent barriers for timeout. - /// - /// A timed-out txn never staged here. It parks on the commit barrier - /// with a `ParticipantError` abort vote and keeps its locks. Its position - /// is marked applied only once the abort verdict drops it. - pub(in crate::control::cluster::calvin::scheduler::driver::core) fn check_dependent_barrier_timeouts( - &mut self, - ) { - // Collect timed-out txn ids first (avoid borrowing issues). - let timed_out: Vec = self - .dependent_barrier - .iter() - .filter(|(_, b)| b.is_timed_out()) - .map(|(id, _)| *id) - .collect(); - - for txn_id in timed_out { - if let Some(barrier) = self.dependent_barrier.remove(&txn_id) { + /// Decide the open barrier of `txn_id` from its log. + fn decide_barrier(&mut self, txn_id: TxnId) { + let Some(barrier) = self.dependent_barrier.get(&txn_id) else { + return; + }; + let empty = BarrierLog::default(); + let log = self.barrier_logs.get(&txn_id).unwrap_or(&empty); + match log.outcome(&barrier.passive) { + BarrierOutcome::Complete => self.complete_barrier(txn_id), + BarrierOutcome::TimedOut => { + let missing = log.missing(&barrier.passive); warn!( vshard_id = self.vshard_id, epoch = txn_id.epoch, position = txn_id.position, - still_waiting = ?barrier.waiting_for, - "calvin: dependent-read barrier timed out; voting abort" + ?missing, + "calvin: the log timed out the dependent-read barrier; voting abort" ); self.metrics.record_executor_error(); self.metrics.record_infra_abort( crate::control::cluster::calvin::scheduler::metrics::infra_abort_reason::PASSIVE_PARTICIPANT_TIMEOUT, ); - let stage_error = format!( - "dependent-read barrier timed out waiting for passive vShards {:?}", - barrier.waiting_for + self.abort_open_barrier( + txn_id, + AbortReason::ParticipantError, + format!( + "the dependent-read barrier timed out in the log before the read \ + results of passive vShards {missing:?}" + ), ); - self.park_unstaged( - barrier.txn, + } + BarrierOutcome::Waiting => { + // An abort verdict is stored already: a peer voted abort. The + // txn stages nothing here, so its locks free at once. + let verdict = self.registry.verdict(nodedb_cluster::calvin::TxnId::new( + txn_id.epoch, + txn_id.position, + )); + if verdict == Some(false) { + self.abort_open_barrier( + txn_id, + AbortReason::ParticipantError, + "an abort verdict reached the txn's open dependent-read barrier".to_owned(), + ); + } + } + } + } + + /// Close the complete barrier of `txn_id`: stage the active slice with + /// the read values, or vote `PredictionDrift` when a value differs from + /// the one the coordinator read. + fn complete_barrier(&mut self, txn_id: TxnId) { + let Some(barrier) = self.dependent_barrier.remove(&txn_id) else { + return; + }; + let injected = self + .barrier_logs + .get(&txn_id) + .map(BarrierLog::injected_reads) + .unwrap_or_default(); + let drift = barrier + .txn + .tx_class + .dependent_reads + .as_ref() + .and_then(|spec| expected_reads_drift(spec, &injected)); + if let Some(detail) = drift { + info!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + %detail, + "calvin: a passive read moved since the coordinator read it; voting prediction drift" + ); + self.park_unstaged( + barrier.txn, + txn_id, + barrier.lock_owner, + AbortReason::PredictionDrift, + detail, + ); + return; + } + self.dispatch_active_txn(barrier.txn, txn_id, barrier.lock_owner, injected); + } + + /// Close the open barrier of `txn_id` with an abort vote for `reason`. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn abort_open_barrier( + &mut self, + txn_id: TxnId, + reason: AbortReason, + detail: String, + ) { + let Some(barrier) = self.dependent_barrier.remove(&txn_id) else { + return; + }; + self.park_unstaged(barrier.txn, txn_id, barrier.lock_owner, reason, detail); + } + + /// Hand the barrier log of every txn this scheduler did not finish back + /// to the vShard's read-result buffer. A stopping scheduler calls it, so + /// the next scheduler of the vShard starts from the same entries. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn return_barrier_logs( + &mut self, + ) { + for (txn_id, log) in std::mem::take(&mut self.barrier_logs) { + if !self.ledger.is_applied(txn_id.epoch, txn_id.position) { + self.read_results.restore( txn_id, - barrier.lock_owner, - nodedb_cluster::calvin::AbortReason::ParticipantError, - stage_error, + BufferedEvents { + memory: log, + stored: false, + }, ); - // A restage under a COMMIT verdict opens a barrier again. It - // starts from what this one received. - self.hold_reads(txn_id, barrier.received); } } } + + /// Remove every stored barrier row of this vShard whose txn finished. + /// A failed pass keeps the rows, and the next stall tick retries. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn sweep_finished_barrier_rows( + &self, + ) { + if let Err(e) = barrier_store::sweep_finished( + self.shared.credentials.catalog(), + self.vshard_id, + &self.ledger, + ) { + warn!( + vshard_id = self.vshard_id, + error = %e, + "calvin: the barrier rows of finished txns were not removed; retrying on \ + the next stall tick" + ); + crate::diag::calvin_barrier_log_store_failed(self.vshard_id, None, "remove", &e); + } + } + + /// Drop every barrier entry of `txn_id`, which finished here: the + /// buffered ones, and its stored row. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn forget_barrier_entries( + &mut self, + txn_id: TxnId, + held_log: bool, + ) { + let buffered = self.read_results.forget_txn(txn_id); + if !held_log && !buffered { + return; + } + // A failed remove leaves the row of a finished txn. No barrier reads + // it, and the stall tick's sweep removes it. + if let Err(e) = + barrier_store::remove_log(self.shared.credentials.catalog(), self.vshard_id, txn_id) + { + warn!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + error = %e, + "calvin: the barrier row of a finished txn was not removed" + ); + crate::diag::calvin_barrier_log_store_failed( + self.vshard_id, + Some((txn_id.epoch, txn_id.position)), + "remove", + &e, + ); + } + } } #[cfg(test)] mod tests { - use std::collections::{BTreeMap, BTreeSet}; - use std::time::{Duration, Instant}; + use std::collections::BTreeMap; + use nodedb_cluster::calvin::types::{DependentReadSpec, SchedulerInput, SequencedTxn}; use nodedb_cluster::calvin::{ - AbortReason, CalvinCompletionRegistry, ParticipantVote, SequencerEntry, VerdictOutcome, + AbortReason, CalvinCompletionRegistry, SequencerEntry, VerdictOutcome, }; + use nodedb_physical::physical_plan::PhysicalPlan; + use nodedb_physical::physical_plan::meta::{MetaOp, PassiveReadKeyId}; + use nodedb_types::{QualifiedCollection, Value}; use super::*; - use crate::control::cluster::calvin::scheduler::driver::barrier::PendingDependentBarrier; + use crate::bridge::dispatch::CoreChannelDataSide; + use crate::control::cluster::calvin::scheduler::BarrierEvent; + use crate::control::cluster::calvin::scheduler::driver::core::process::LedgerMark; use crate::control::cluster::calvin::scheduler::driver::core::test_proposer::{ CapturingProposer, lead_data_group, }; use crate::control::cluster::calvin::scheduler::driver::core::test_support::{ - build_test_scheduler_with_data_side, make_local_write_txn, make_sequenced_txn, - test_coll_vshard, + build_test_scheduler_with_data_side, make_local_write_txn, test_coll_vshard, }; use crate::control::cluster::calvin::scheduler::driver::types::CommitState; - const VSHARD: u32 = 7; + /// The passive vShard of every test txn: not the scheduler's own. + const PASSIVE: u32 = 9; - /// A timed-out barrier votes abort and parks. Its vote completes the - /// tally into an abort verdict, and the position stays unapplied until - /// that verdict drops the txn. - #[tokio::test] - async fn a_timed_out_barrier_votes_a_complete_abort_tally() { + fn row() -> PassiveReadKeyId { + PassiveReadKeyId::kv( + QualifiedCollection::from_stored("items".to_owned()), + b"alice:sword".to_vec(), + ) + } + + /// A dependent txn that writes `test_coll` and reads `row()` on + /// [`PASSIVE`], whose coordinator read `expected` for it. + fn dependent_txn(epoch: u64, expected: Option<&[u8]>) -> SequencedTxn { + let mut txn = make_local_write_txn(epoch, 0); + txn.tx_class.dependent_reads = Some(DependentReadSpec { + passive_reads: BTreeMap::from([(PASSIVE, Vec::new())]), + expected: BTreeMap::from([(row(), expected.map(<[u8]>::to_vec))]), + }); + txn + } + + fn read(value: &[u8]) -> BarrierEvent { + BarrierEvent::Read { + passive_vshard: PASSIVE, + values: vec![(row(), Value::Bytes(value.to_vec()))], + } + } + + /// A scheduler on the `test_coll` vShard with a capturing proposer. Not + /// a leader until the test makes it one. + fn scheduler() -> ( + Scheduler, + tempfile::TempDir, + CoreChannelDataSide, + Arc, + Arc, + ) { let registry = CalvinCompletionRegistry::new_detached(); - let (mut scheduler, _dir, _data_side) = - build_test_scheduler_with_data_side(VSHARD, std::sync::Arc::clone(®istry)); + let (mut scheduler, dir, data_side) = + build_test_scheduler_with_data_side(test_coll_vshard(), Arc::clone(®istry)); let proposer = CapturingProposer::accepting(); scheduler.sequencer_proposer = proposer.clone(); + (scheduler, dir, data_side, proposer, registry) + } + + fn buffer(scheduler: &Scheduler, epoch: u64, event: BarrierEvent) { + scheduler + .read_results + .push(TxnId::new(epoch, 0), event, 64, true); + } + + fn grant(scheduler: &mut Scheduler, txn: SequencedTxn) { + scheduler.process_scheduler_input(SchedulerInput::Txn(Box::new(txn))); + } + + /// Whether the Data Plane received the active stage of `(epoch, 0)`. + fn staged_active(data_side: &mut CoreChannelDataSide, epoch: u64) -> bool { + let mut staged = false; + while let Ok(request) = data_side.request_rx.try_pop() { + staged |= matches!( + request.inner.plan, + PhysicalPlan::Meta(MetaOp::CalvinExecuteActive { epoch: e, position: 0, .. }) + if e == epoch + ); + } + staged + } + + fn abort_votes(proposer: &CapturingProposer) -> Vec { + proposer + .accepted() + .into_iter() + .filter_map(|entry| match entry { + SequencerEntry::AbortVote { reason, .. } => Some(reason), + _ => None, + }) + .collect() + } + + /// A read result the log applied before this node granted the txn waits + /// in the buffer. The grant opens the barrier from it, and the active + /// slice stages at once. + #[tokio::test] + async fn a_result_buffered_before_the_grant_completes_the_barrier() { + let (mut scheduler, _dir, mut data_side, proposer, _registry) = scheduler(); lead_data_group(&mut scheduler); - let txn_id = TxnId::new(40, 0); - let cluster_txn = nodedb_cluster::calvin::TxnId::new(40, 0); - registry.seed_expected(cluster_txn, 1); - scheduler.dependent_barrier.insert( - txn_id, - PendingDependentBarrier { - txn: make_sequenced_txn(40, 0), - lock_owner: txn_id, - waiting_for: BTreeSet::from([9]), - received: BTreeMap::new(), - // no-determinism: test-only deadline already passed. - timeout_at: Instant::now() - Duration::from_millis(1), - }, - ); + buffer(&scheduler, 50, read(b"v1")); + + grant(&mut scheduler, dependent_txn(50, Some(b"v1"))); - scheduler.check_dependent_barrier_timeouts(); + assert!(staged_active(&mut data_side, 50)); assert!(scheduler.dependent_barrier.is_empty()); + assert_eq!(scheduler.read_results.waiting_txns(), 0); assert_eq!( - scheduler.pending.get(&txn_id).map(|p| p.commit_state), - Some(CommitState::AwaitingVerdict) - ); - assert!(!scheduler.applied.is_applied(40, 0)); - assert_eq!( - proposer.accepted(), - vec![SequencerEntry::AbortVote { - epoch: 40, - position: 0, - vshard: VSHARD, - reason: AbortReason::ParticipantError, - }] + scheduler + .pending + .get(&TxnId::new(50, 0)) + .map(|p| p.commit_state), + Some(CommitState::Staged) ); + assert!(abort_votes(&proposer).is_empty()); + } - registry.note_vote( - cluster_txn, - VSHARD, - ParticipantVote::Abort(AbortReason::ParticipantError), - ); - assert_eq!( - registry.drain_unproposed_verdicts(), - vec![( - cluster_txn, - VerdictOutcome::Abort(AbortReason::ParticipantError) - )] - ); - registry.note_verdict( - cluster_txn, - VerdictOutcome::Abort(AbortReason::ParticipantError), - ); - scheduler.resume_on_verdict(txn_id, false); - assert_eq!( - scheduler.pending.get(&txn_id).map(|p| p.commit_state), - Some(CommitState::AwaitingDrop) + /// A restarted node finds a txn's result in its stored row alone. The + /// grant opens the barrier from the row, the active slice stages, and + /// the txn's finish removes the row. + #[tokio::test] + async fn a_stored_result_completes_the_barrier_after_a_restart() { + let (mut scheduler, _dir, mut data_side, proposer, _registry) = scheduler(); + lead_data_group(&mut scheduler); + let txn_id = TxnId::new(51, 0); + let catalog = scheduler.shared.credentials.catalog(); + barrier_store::save_event(catalog, scheduler.vshard_id, txn_id, &read(b"v1")) + .expect("store the result"); + scheduler.read_results.mark_stored([txn_id]); + + grant(&mut scheduler, dependent_txn(51, Some(b"v1"))); + + assert!(staged_active(&mut data_side, 51)); + assert!(scheduler.dependent_barrier.is_empty()); + assert_eq!(scheduler.read_results.waiting_txns(), 0); + assert!(abort_votes(&proposer).is_empty()); + + let lock_owner = scheduler + .pending + .get(&txn_id) + .map(|pending| pending.lock_owner) + .expect("the staged txn is pending"); + scheduler.release_and_mark_applied(txn_id, lock_owner, LedgerMark::ByApply); + assert!( + scheduler + .shared + .credentials + .catalog() + .load_calvin_barrier_log(scheduler.vshard_id, 51, 0) + .expect("load") + .is_none(), + "the finished txn's row is removed" ); - assert!(!scheduler.applied.is_applied(40, 0)); } - /// A barrier that timed out keeps what it received, and a read result - /// that lands after the timeout is kept too. Under a COMMIT verdict the - /// restage opens a barrier from those reads. They complete it, so the - /// active stage dispatches at once. + /// A result pushed while the barrier is open completes it on the drain. #[tokio::test] - async fn a_restaged_barrier_starts_from_the_reads_it_received() { - use nodedb_cluster::calvin::types::DependentReadSpec; - use nodedb_physical::physical_plan::PhysicalPlan; - use nodedb_physical::physical_plan::meta::MetaOp; + async fn a_result_after_the_grant_completes_the_open_barrier() { + let (mut scheduler, _dir, mut data_side, _proposer, _registry) = scheduler(); + lead_data_group(&mut scheduler); + grant(&mut scheduler, dependent_txn(51, Some(b"v1"))); + assert!(scheduler.dependent_barrier.contains_key(&TxnId::new(51, 0))); - let registry = CalvinCompletionRegistry::new_detached(); - let (mut scheduler, _dir, mut data_side) = build_test_scheduler_with_data_side( - test_coll_vshard(), - std::sync::Arc::clone(®istry), - ); + buffer(&scheduler, 51, read(b"v1")); + scheduler.drain_read_results(); + + assert!(staged_active(&mut data_side, 51)); + assert!(scheduler.dependent_barrier.is_empty()); + } + + /// The timeout entry above every result changes nothing: the barrier + /// completed in the log first. + #[tokio::test] + async fn a_timeout_after_every_result_is_ignored() { + let (mut scheduler, _dir, mut data_side, proposer, _registry) = scheduler(); lead_data_group(&mut scheduler); - let txn_id = TxnId::new(41, 0); - let mut txn = make_local_write_txn(41, 0); - txn.tx_class.dependent_reads = Some(DependentReadSpec { - passive_reads: BTreeMap::from([(9, Vec::new()), (10, Vec::new())]), - }); - scheduler.dependent_barrier.insert( - txn_id, - PendingDependentBarrier { - txn, - lock_owner: txn_id, - waiting_for: BTreeSet::from([10]), - received: BTreeMap::from([(9, Vec::new())]), - // no-determinism: test-only deadline already passed. - timeout_at: Instant::now() - Duration::from_millis(1), - }, - ); + buffer(&scheduler, 52, read(b"v1")); + buffer(&scheduler, 52, BarrierEvent::Timeout); + + grant(&mut scheduler, dependent_txn(52, Some(b"v1"))); + + assert!(staged_active(&mut data_side, 52)); + assert!(abort_votes(&proposer).is_empty()); + } + + /// A timeout entry below a result times the barrier out: the leader + /// votes abort, and the late result does not count. + #[tokio::test] + async fn a_timeout_before_the_result_votes_abort() { + let (mut scheduler, _dir, mut data_side, proposer, _registry) = scheduler(); + lead_data_group(&mut scheduler); + grant(&mut scheduler, dependent_txn(53, Some(b"v1"))); - scheduler.check_dependent_barrier_timeouts(); + buffer(&scheduler, 53, BarrierEvent::Timeout); + buffer(&scheduler, 53, read(b"v1")); + scheduler.drain_read_results(); + + assert!(!staged_active(&mut data_side, 53)); assert_eq!( - scheduler.pending.get(&txn_id).map(|p| p.commit_state), + scheduler + .pending + .get(&TxnId::new(53, 0)) + .map(|p| p.commit_state), Some(CommitState::AwaitingVerdict) ); - scheduler.handle_read_result(ReadResultEvent { - epoch: 41, - position: 0, - passive_vshard: 10, - tenant_id: crate::types::TenantId::new(1), - values: Vec::new(), - }); + assert_eq!(abort_votes(&proposer), vec![AbortReason::ParticipantError]); + assert!(!scheduler.applied.is_applied(53, 0)); + } + + /// A value that moved since the coordinator read it votes + /// `PredictionDrift` and stages nothing. + #[tokio::test] + async fn a_moved_value_votes_prediction_drift() { + let (mut scheduler, _dir, mut data_side, proposer, _registry) = scheduler(); + lead_data_group(&mut scheduler); + buffer(&scheduler, 54, read(b"v2")); + + grant(&mut scheduler, dependent_txn(54, Some(b"v1"))); + + assert!(!staged_active(&mut data_side, 54)); + assert_eq!(abort_votes(&proposer), vec![AbortReason::PredictionDrift]); + } + + /// A follower holds the log of a dependent txn its leader committed. The + /// result reached the follower before its grant, and the follower waited + /// past any timeout of its own. Promoted, it opens the barrier from the + /// log, stages the slice, and neither votes abort nor halts. + #[tokio::test] + async fn a_promoted_follower_stages_the_committed_txn_without_halting() { + let (mut scheduler, _dir, mut data_side, proposer, registry) = scheduler(); + scheduler.config.dependent_read_passive_timeout_ms = 20; + buffer(&scheduler, 55, read(b"v1")); + grant(&mut scheduler, dependent_txn(55, Some(b"v1"))); + let txn_id = TxnId::new(55, 0); assert_eq!( - scheduler - .held_reads - .get(&txn_id) - .map(|held| held.keys().copied().collect::>()), - Some(vec![9, 10]) + scheduler.pending.get(&txn_id).map(|p| p.commit_state), + Some(CommitState::Following) ); - + // The earlier leader voted commit and the verdict is stored. registry.note_verdict( - nodedb_cluster::calvin::TxnId::new(41, 0), + nodedb_cluster::calvin::TxnId::new(55, 0), VerdictOutcome::Commit, ); scheduler.resume_on_verdict(txn_id, true); + // The leader's barrier timeout passes on the follower's clock. + // no-determinism: test-only pause past the configured timeout. + tokio::time::sleep(scheduler.config.passive_timeout()).await; + + lead_data_group(&mut scheduler); + + assert!(staged_active(&mut data_side, 55)); + assert!(scheduler.apply_halt().is_none()); + assert!(abort_votes(&proposer).is_empty()); assert_eq!( scheduler.pending.get(&txn_id).map(|p| p.commit_state), - Some(CommitState::AwaitingRestage) + Some(CommitState::Staged) ); - if let Some(restage) = scheduler.restages.get_mut(&txn_id) { - // no-determinism: test-only backoff end in the past. - restage.due = Some(Instant::now() - Duration::from_millis(1)); - } - scheduler.restage_due(); + } + + /// A follower whose log timed the barrier out votes abort once + /// promoted: the decision the earlier leader reached from the same log. + #[tokio::test] + async fn a_promoted_follower_decides_a_logged_timeout_alike() { + let (mut scheduler, _dir, mut data_side, proposer, _registry) = scheduler(); + grant(&mut scheduler, dependent_txn(56, Some(b"v1"))); + buffer(&scheduler, 56, BarrierEvent::Timeout); + scheduler.drain_read_results(); + buffer(&scheduler, 56, read(b"v1")); + scheduler.drain_read_results(); + + lead_data_group(&mut scheduler); + + assert!(!staged_active(&mut data_side, 56)); + assert_eq!(abort_votes(&proposer), vec![AbortReason::ParticipantError]); + } + + /// An abort verdict ends an open barrier: the txn drops with no stage. + #[tokio::test] + async fn an_abort_verdict_ends_an_open_barrier() { + let (mut scheduler, _dir, mut data_side, _proposer, registry) = scheduler(); + lead_data_group(&mut scheduler); + grant(&mut scheduler, dependent_txn(57, Some(b"v1"))); + let txn_id = TxnId::new(57, 0); + registry.note_verdict( + nodedb_cluster::calvin::TxnId::new(57, 0), + VerdictOutcome::Abort(AbortReason::ParticipantError), + ); + + scheduler.resume_on_verdict(txn_id, false); - let mut staged_active = false; - while let Ok(request) = data_side.request_rx.try_pop() { - staged_active |= matches!( - request.inner.plan, - PhysicalPlan::Meta(MetaOp::CalvinExecuteActive { - epoch: 41, - position: 0, - .. - }) - ); - } - assert!(staged_active, "the restaged barrier completes at once"); assert!(scheduler.dependent_barrier.is_empty()); - assert!(scheduler.held_reads.is_empty()); + assert!(!staged_active(&mut data_side, 57)); assert_eq!( scheduler.pending.get(&txn_id).map(|p| p.commit_state), - Some(CommitState::Staged) + Some(CommitState::AwaitingDrop) ); } + + /// The leader proposes the barrier's timeout entry once its timeout + /// passes, and decides nothing until the entry applies. + #[tokio::test] + async fn the_leader_proposes_a_due_timeout_and_waits_for_the_log() { + let (mut scheduler, _dir, _data_side, proposer, _registry) = scheduler(); + lead_data_group(&mut scheduler); + grant(&mut scheduler, dependent_txn(58, Some(b"v1"))); + let txn_id = TxnId::new(58, 0); + let group_id = { + let mr = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()); + mr.routing() + .read() + .unwrap_or_else(|p| p.into_inner()) + .group_for_vshard(scheduler.vshard_id) + .expect("routed vShard") + }; + let term_start = scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .term_start_index(group_id) + .expect("term start"); + if let Some(barrier) = scheduler.dependent_barrier.get_mut(&txn_id) { + // no-determinism: test-only deadline already passed. + barrier.timeout_due = std::time::Instant::now() - std::time::Duration::from_millis(1); + } + + scheduler.propose_due_read_timeouts(); + + assert!( + scheduler + .multi_raft + .lock() + .unwrap_or_else(|p| p.into_inner()) + .log_term_at(group_id, term_start + 1) + .is_some(), + "the timeout entry is in the group's log" + ); + let barrier = scheduler + .dependent_barrier + .get(&txn_id) + .expect("the barrier stays open until the entry applies"); + assert!(barrier.timeout_due > std::time::Instant::now()); + assert!(abort_votes(&proposer).is_empty()); + } + + /// A stopping scheduler hands the log of an unfinished txn back to the + /// buffer for the next scheduler of the vShard. + #[tokio::test] + async fn a_stopping_scheduler_hands_its_logs_back() { + let (mut scheduler, _dir, _data_side, _proposer, _registry) = scheduler(); + grant(&mut scheduler, dependent_txn(59, Some(b"v1"))); + buffer(&scheduler, 59, read(b"v1")); + scheduler.drain_read_results(); + assert_eq!(scheduler.read_results.waiting_txns(), 0); + + scheduler.return_barrier_logs(); + + assert_eq!(scheduler.read_results.waiting_txns(), 1); + assert!(scheduler.barrier_logs.is_empty()); + } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_timeout.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_timeout.rs new file mode 100644 index 000000000..388b06511 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/read_timeout.rs @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The data-group leader proposes the timeout entry of a dependent-read +//! barrier that waited too long. +//! +//! The leader's own clock decides only when it proposes +//! `ReplicatedWrite::CalvinReadTimeout`. The entry's place in the vShard's +//! log decides what it does (see [`super::read_result`]): every replica +//! applies it after the same read results. The leader proposes straight to +//! its group with `MultiRaft::propose`, as it proposes a redo: the write +//! gate exempts the entry, which writes no row. +//! +//! A proposal re-arms the barrier's timeout. An entry a leader change +//! lost is proposed again then, and a duplicate that lands after the first +//! changes nothing. + +use std::time::Instant; + +use nodedb_cluster::ClusterError; +use nodedb_raft::RaftError; + +use super::scheduler::Scheduler; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; +use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite}; + +/// Why a timeout entry did not reach the log. +enum TimeoutRefusal { + /// This node does not lead the vShard's data group. + NotLeader, + /// The entry cannot be built, or the group refused it now. + Failed(String), +} + +impl Scheduler { + /// Propose the timeout entry of every open barrier whose timeout passed. + /// Runs on the leader only. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn propose_due_read_timeouts( + &mut self, + ) { + if !self.role.is_leader() || self.dependent_barrier.is_empty() { + return; + } + // no-determinism: the leader's clock decides only when it proposes the timeout entry; the entry's place in the log decides the barrier. + let now = Instant::now(); + let due: Vec = self + .dependent_barrier + .iter() + .filter(|(_, barrier)| barrier.timeout_due <= now) + .map(|(txn_id, _)| *txn_id) + .collect(); + for txn_id in due { + match self.propose_read_timeout(txn_id) { + Ok(()) => {} + Err(TimeoutRefusal::NotLeader) => { + self.on_not_leader(); + return; + } + Err(TimeoutRefusal::Failed(error)) => tracing::debug!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + %error, + "calvin: barrier timeout entry not proposed; proposing it again later" + ), + } + if let Some(barrier) = self.dependent_barrier.get_mut(&txn_id) { + barrier.timeout_due = now + self.config.passive_timeout(); + } + } + } + + /// The earliest instant this leader proposes a barrier's timeout entry, + /// when it holds a barrier open. + pub(in crate::control::cluster::calvin::scheduler::driver::core) fn next_read_timeout_due( + &self, + ) -> Option { + if !self.role.is_leader() { + return None; + } + self.dependent_barrier + .values() + .map(|barrier| barrier.timeout_due) + .min() + } + + /// Append the timeout entry of `txn_id`'s barrier to this vShard's + /// data-group log. + fn propose_read_timeout(&self, txn_id: TxnId) -> Result<(), TimeoutRefusal> { + let Some(barrier) = self.dependent_barrier.get(&txn_id) else { + return Ok(()); + }; + let tenant_id = barrier.txn.tx_class.tenant_id.as_u64(); + let entry = ReplicatedEntry::new( + tenant_id, + barrier.txn.tx_class.database_id.as_u64(), + self.vshard_id, + ReplicatedWrite::CalvinReadTimeout { + epoch: txn_id.epoch, + position: txn_id.position, + tenant_id, + }, + ); + let bytes = entry + .encode() + .map_err(|error| TimeoutRefusal::Failed(error.to_string()))?; + let mut mr = self.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); + match mr.propose(self.vshard_id, bytes) { + Ok(_) => { + tracing::info!( + vshard_id = self.vshard_id, + epoch = txn_id.epoch, + position = txn_id.position, + "calvin: dependent-read barrier waited past its timeout; proposed the \ + timeout entry" + ); + Ok(()) + } + Err(ClusterError::Raft(RaftError::NotLeader { .. })) => Err(TimeoutRefusal::NotLeader), + Err(error) => Err(TimeoutRefusal::Failed(error.to_string())), + } + } +} diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_applied.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_applied.rs index 904be3dd5..e9efa74d0 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_applied.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_applied.rs @@ -158,11 +158,12 @@ fn holds_staged_state(state: CommitState) -> bool { | CommitState::AwaitingRedoResolve | CommitState::AwaitingDrop => true, // The install consumed the staged entry of a proposed slice, a - // follower staged nothing, and a txn awaiting its restage discarded - // what it staged. + // follower staged nothing, a txn awaiting its restage discarded what + // it staged, and a passive read stages nothing. CommitState::AwaitingRedoApply { .. } | CommitState::Following - | CommitState::AwaitingRestage => false, + | CommitState::AwaitingRestage + | CommitState::ReadingPassive => false, } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs index b1fd6d0c5..719576029 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs @@ -146,6 +146,7 @@ impl Scheduler { }); if held { tracing::info!( + node_id = self.shared.node_id, vshard_id = self.vshard_id, epoch = txn_id.epoch, position = txn_id.position, @@ -253,7 +254,8 @@ impl Scheduler { | CommitState::AwaitingResolveTurn | CommitState::AwaitingRedoResolve | CommitState::AwaitingDrop - | CommitState::AwaitingRestage => None, + | CommitState::AwaitingRestage + | CommitState::ReadingPassive => None, }) .collect(); for (txn_id, proposed) in owed { diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/restage.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/restage.rs index 3d25c9e54..aad369e37 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/restage.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/restage.rs @@ -161,13 +161,13 @@ impl Scheduler { (!self.has_deferred_for(txn_id)).then_some(at) } - /// Forget the restage and the held reads of `txn_id`, which finished. + /// Forget the restage and the barrier log of `txn_id`, which finished. pub(in crate::control::cluster::calvin::scheduler::driver::core) fn forget_held_state( &mut self, txn_id: TxnId, ) { self.restages.remove(&txn_id); - self.held_reads.remove(&txn_id); + self.barrier_logs.remove(&txn_id); } /// Stage `txn_id` again through the grant path. diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/role.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/role.rs index e3d085ec7..ae1f2a614 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/role.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/role.rs @@ -98,6 +98,19 @@ impl Scheduler { } self.role.term = Some(leadership.term); self.role.term_start = leadership.term_start; + tracing::debug!( + node_id = self.shared.node_id, + vshard_id = self.vshard_id, + group_id = leadership.group_id, + term = leadership.term, + term_start = ?leadership.term_start, + applied = self + .shared + .applied_index_watcher(leadership.group_id) + .current(), + "calvin scheduler: this node won the data group; staging waits for the term's \ + first entry to apply" + ); } if self.role.is_leader() { return; @@ -143,6 +156,7 @@ impl Scheduler { fn promote(&mut self) { self.role.role = Role::Leader; tracing::info!( + node_id = self.shared.node_id, vshard_id = self.vshard_id, term = self.role.term, "calvin scheduler: this node leads the vShard; staging its held txns" @@ -169,6 +183,7 @@ impl Scheduler { ) { self.role.role = Role::Follower; tracing::info!( + node_id = self.shared.node_id, vshard_id = self.vshard_id, reason = why, "calvin scheduler: this node stops leading the vShard; holding its txns" @@ -185,23 +200,36 @@ impl Scheduler { // Only a leader restages. A later term counts its own restages. self.restages.clear(); // A barrier waits only on the leader. Its txn holds its locks and - // follows the log like any other. It keeps the reads it received, - // so a later barrier of it starts from them. + // follows the log like any other. Its barrier log stays, so a later + // barrier of it decides from the same entries. let barriers: Vec = self.dependent_barrier.keys().copied().collect(); for txn_id in barriers { if let Some(barrier) = self.dependent_barrier.remove(&txn_id) { self.follow(barrier.txn, txn_id, barrier.lock_owner); - self.hold_reads(txn_id, barrier.received); } } } /// Hold `txn_id` `Following`, and discard what this node staged for it. + /// + /// The txn's scope becomes its slice's own, from its plans. A leader's + /// step can hold a narrower one: a passive read or a parked abort vote + /// holds a scope that writes nothing. A follower that kept it would end a + /// COMMIT with no entry, mark the position applied, and then skip the + /// slice's redo when it applies. fn release_to_following(&mut self, txn_id: TxnId) { + let Some(scope) = self + .pending + .get(&txn_id) + .map(|pending| self.local_slice_scope(&pending.txn, txn_id)) + else { + return; + }; let Some(pending) = self.pending.get_mut(&txn_id) else { return; }; pending.commit_state = CommitState::Following; + pending.scope = scope; pending.awaiting = None; pending.verdict_deadline = None; pending.stage_error = None; @@ -243,7 +271,8 @@ fn leader_drives(state: CommitState) -> bool { | CommitState::AwaitingResolveTurn | CommitState::AwaitingRedoResolve | CommitState::AwaitingRedoApply { .. } - | CommitState::AwaitingRestage => true, + | CommitState::AwaitingRestage + | CommitState::ReadingPassive => true, // A drop ends the txn with no log entry whatever the role. CommitState::AwaitingDrop | CommitState::Following => false, } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs index c51fc702c..f4fa916db 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs @@ -36,6 +36,8 @@ impl Scheduler { let capacity_freed = Arc::clone(&self.capacity_freed); // Woken when the data-group apply loop concludes a stamped redo. let inbox = Arc::clone(self.inbox.inbox()); + // Woken when the data-group apply loop buffers a barrier entry. + let read_results = Arc::clone(&self.read_results); // Woken when a cut's barrier is proposed or applied on this node. let shared = Arc::clone(&self.shared); // Set when a tick left armed catch-up unreplayed. The next open-gate @@ -65,7 +67,8 @@ impl Scheduler { if self.release_cut_holds() { self.retry_redo_proposals(); } - self.check_dependent_barrier_timeouts(); + self.drain_read_results(); + self.propose_due_read_timeouts(); self.check_awaiting_verdict_stalls(); self.restage_due(); self.resume_metadata_hold(); @@ -80,6 +83,10 @@ impl Scheduler { } // The backoff end of the next restage, when one waits. let restage_at = self.next_restage_at().map(tokio::time::Instant::from_std); + // When this leader next proposes a barrier's timeout entry. + let read_timeout_at = self + .next_read_timeout_due() + .map(tokio::time::Instant::from_std); tokio::select! { biased; @@ -117,12 +124,10 @@ impl Scheduler { self.handle_verdict_signal(signal); } - maybe_event = self.read_result_rx.recv() => { - let Some(event) = maybe_event else { - self.log_superseded("read result"); - break; - }; - self.handle_read_result(event); + // The apply loop buffered a barrier entry. Drained in every + // state, so a held txn keeps every entry of its log. + () = read_results.pushed() => { + self.drain_read_results(); } maybe_promoted = self.promotion_rx.recv() => { @@ -150,6 +155,13 @@ impl Scheduler { // restage backoff ended. } + _ = tokio::time::sleep_until( + read_timeout_at.unwrap_or_else(tokio::time::Instant::now) + ), if read_timeout_at.is_some() => { + // The next loop pass proposes the timeout entry of every + // barrier whose timeout passed. + } + _ = tokio::time::sleep(ROLE_POLL), if self.role.awaits_term_start() => { // The next loop pass checks whether the term's no-op // applied, and stages the held txns once it has. @@ -193,13 +205,25 @@ impl Scheduler { // install already reported is never proposed again. self.drain_inbox(); self.retry_redo_proposals(); + // A barrier entry the apply loop buffered after the + // scheduler finished its txn is spent. + let ledger = Arc::clone(&self.ledger); + self.read_results.discard_finished(|txn| { + ledger.is_applied(txn.epoch, txn.position) + }); + // So is a stored row of a finished txn its finish did + // not remove. + self.sweep_finished_barrier_rows(); // The top-of-loop check_awaiting_verdict_stalls / - // check_dependent_barrier_timeouts and the deferred re-send + // propose_due_read_timeouts and the deferred re-send // pass run on every wake; this arm guarantees the loop wakes // to run them (and the drain) when no other event arrives. } } } + // The next scheduler of the vShard starts from the barrier entries + // this one took. + self.return_barrier_logs(); } /// Log that a scheduler channel closed and the loop exits. diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs index 94189c3b5..089354880 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/scheduler.rs @@ -11,7 +11,7 @@ use nodedb_cluster::MultiRaft; use nodedb_cluster::calvin::types::SchedulerInput; use nodedb_cluster::calvin::{CalvinCompletionRegistry, SequencerStateMachine, VerdictSignal}; -use super::super::barrier::{PendingDependentBarrier, ReadResultEvent}; +use super::super::barrier::PendingDependentBarrier; use super::super::config::SchedulerConfig; use super::super::types::{BlockedTxn, PendingTxn}; use super::deferred::DeferredQueue; @@ -98,23 +98,23 @@ pub struct Scheduler { /// Blocked transactions awaiting lock release. pub(in crate::control::cluster::calvin::scheduler::driver::core) blocked: BTreeMap, - /// Dependent-read barriers awaiting passive read results. - /// `BTreeMap` for determinism. + /// Dependent-read barriers this leader holds open. `BTreeMap` for + /// determinism. pub(in crate::control::cluster::calvin::scheduler::driver::core) dependent_barrier: BTreeMap, - /// Passive read results of dependent txns in `pending` that wait at no - /// barrier. A later barrier of the txn starts from them. Each entry - /// leaves when its txn completes. - pub(in crate::control::cluster::calvin::scheduler::driver::core) held_reads: - BTreeMap, + /// The barrier entries of the data-group log, folded per dependent txn + /// this scheduler holds: at a barrier, or in `pending`. A barrier of the + /// txn decides from them. Each entry leaves when its txn completes. + pub(in crate::control::cluster::calvin::scheduler::driver::core) barrier_logs: + BTreeMap, + /// This vShard's read-result buffer: the barrier entries the apply loop + /// applied for txns this scheduler did not hold yet. + pub(in crate::control::cluster::calvin::scheduler::driver::core) read_results: + Arc, /// Committed txns whose stage failed on this leader, with their /// restage count and backoff. See [`super::restage`]. pub(in crate::control::cluster::calvin::scheduler::driver::core) restages: BTreeMap, - /// Channel receiving `CalvinReadResult` Raft apply events from the - /// per-vshard data Raft apply loop. Bounded. - pub(in crate::control::cluster::calvin::scheduler::driver::core) read_result_rx: - mpsc::Receiver, /// Exactly-once applied gate: the fully-applied watermark plus the set of /// applied `(epoch, position)` pairs above it. Replaces a bare per-epoch /// counter so a multi-position epoch is never marked applied on the strength @@ -232,7 +232,6 @@ pub struct SchedulerParams { pub rebuild_target_epoch: u64, pub config: SchedulerConfig, pub metrics: Arc, - pub read_result_rx: mpsc::Receiver, /// The shared lock table for this vShard. Constructed by /// `reconcile_vshard_schedulers` and registered in /// `SharedState.calvin.lock_managers` under the SAME `Arc` passed here. @@ -264,7 +263,6 @@ impl Scheduler { rebuild_target_epoch, config, metrics, - read_result_rx, lock_manager, promotion_rx, registry, @@ -286,6 +284,7 @@ impl Scheduler { .calvin .inboxes .register(vshard_id, config.max_inflight_backlog); + let read_results = shared.calvin.read_results.vshard(vshard_id); let capacity_freed = shared .dispatcher @@ -308,9 +307,9 @@ impl Scheduler { inbox, blocked: BTreeMap::new(), dependent_barrier: BTreeMap::new(), - held_reads: BTreeMap::new(), + barrier_logs: BTreeMap::new(), + read_results, restages: BTreeMap::new(), - read_result_rx, applied: AppliedGate::new(fully_applied_epoch, applied_tail), cut_floors: Default::default(), cut_holds: Default::default(), diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs index 764cc57fa..29102a0a4 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/test_support.rs @@ -23,7 +23,6 @@ use crate::bridge::envelope::{ Admission, ErrorCode, ExemptReason, Payload, Priority, Request, Response, StageVote, Status, }; use crate::control::cluster::calvin::scheduler::SchedulerConfig; -use crate::control::cluster::calvin::scheduler::driver::barrier::ReadResultEvent; use crate::control::cluster::calvin::scheduler::driver::core::scheduler::{ Scheduler, SchedulerParams, }; @@ -63,7 +62,6 @@ pub(super) fn build_test_scheduler(vshard_id: u32) -> (Scheduler, tempfile::Temp ))); let (_tx, receiver) = tokio::sync::mpsc::channel(16); - let (_rr_tx, read_result_rx) = tokio::sync::mpsc::channel(16); let (_prom_tx, promotion_rx) = tokio::sync::mpsc::unbounded_channel(); let (verdict_tx, verdict_rx) = tokio::sync::mpsc::channel(16); registry.register_verdict_signal_sender(vshard_id, verdict_tx); @@ -87,7 +85,6 @@ pub(super) fn build_test_scheduler(vshard_id: u32) -> (Scheduler, tempfile::Temp rebuild_target_epoch: 0, config: SchedulerConfig::default(), metrics: SchedulerMetrics::new(), - read_result_rx, lock_manager, promotion_rx, registry, @@ -122,7 +119,6 @@ pub(super) fn build_test_scheduler_with_data_side( ))); let (_tx, receiver) = tokio::sync::mpsc::channel(16); - let (_rr_tx, read_result_rx) = tokio::sync::mpsc::channel(16); let (_prom_tx, promotion_rx) = tokio::sync::mpsc::unbounded_channel(); let (verdict_tx, verdict_rx) = tokio::sync::mpsc::channel(16); registry.register_verdict_signal_sender(vshard_id, verdict_tx); @@ -140,7 +136,6 @@ pub(super) fn build_test_scheduler_with_data_side( rebuild_target_epoch: 0, config: SchedulerConfig::default(), metrics: SchedulerMetrics::new(), - read_result_rx, lock_manager, promotion_rx, registry, @@ -379,7 +374,6 @@ pub(super) struct RunningScheduler { shutdown: ShutdownWatch, handle: tokio::task::JoinHandle<()>, input_tx: mpsc::Sender, - _read_result_tx: mpsc::Sender, _promotion_tx: mpsc::UnboundedSender>, } @@ -411,10 +405,8 @@ impl RunningScheduler { /// liveness tick. pub(super) fn spawn_scheduler_loop(mut scheduler: Scheduler) -> RunningScheduler { let (input_tx, input_rx) = mpsc::channel(16); - let (read_result_tx, read_result_rx) = mpsc::channel(16); let (promotion_tx, promotion_rx) = mpsc::unbounded_channel(); scheduler.receiver = input_rx; - scheduler.read_result_rx = read_result_rx; scheduler.promotion_rx = promotion_rx; // The loop's liveness tick fires every quarter of this interval. scheduler.config.verdict_stall_warn_ms = 200; @@ -425,7 +417,6 @@ pub(super) fn spawn_scheduler_loop(mut scheduler: Scheduler) -> RunningScheduler shutdown, handle, input_tx, - _read_result_tx: read_result_tx, _promotion_tx: promotion_tx, } } diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/mod.rs index 5baf6021c..6ac5f56ca 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/mod.rs @@ -6,7 +6,6 @@ pub mod core; pub mod helpers; pub mod types; -pub use barrier::ReadResultEvent; pub use config::SchedulerConfig; pub use core::{ CalvinReadResultProposal, RaftSequencerProposer, Scheduler, SchedulerParams, SequencerProposer, diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs index 4fdcda2d9..8de4414e7 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs @@ -184,6 +184,11 @@ pub(in crate::control::cluster::calvin::scheduler::driver) enum CommitState { /// Awaiting the validate-and-stage response, whose `stage_vote` carries /// the local commit vote. Staged, + /// A dependent-read txn whose passive read this leader dispatched. The + /// read stages nothing. Its answer is proposed to every active vShard's + /// data group. A passive-only vShard then votes on it, and an active + /// vShard opens its barrier. + ReadingPassive, /// The staged txn has cast its local vote and PARKED, awaiting the durable /// authoritative GLOBAL verdict aggregated across all participant vShards. /// diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs index dc442031d..d4b0018a5 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/introspection.rs @@ -1,16 +1,47 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Read-only inspection of lock manager state — readiness checks and the -//! test-only counters used to assert on table/holder sizes. +//! Read-only inspection of lock manager state: readiness checks, the keys +//! that block a request, and the test-only counters used to assert on +//! table and holder sizes. use std::collections::BTreeMap; use crate::control::cluster::calvin::scheduler::lock::lock_entry::LockMode; use crate::control::cluster::calvin::scheduler::lock::lock_key::{LockKey, TxnId}; +use super::classify::KeyState; use super::types::LockManager; +/// One key of a lock request that another transaction blocks: the mode it +/// asks, the mode its holders hold, the holders, and the waiters ahead. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KeyContention { + pub key: LockKey, + pub requested: LockMode, + pub held: LockMode, + pub holders: Vec, + pub waiters: Vec<(TxnId, LockMode)>, +} + impl LockManager { + /// Every key of `keys` that `txn` cannot take now, with what blocks it. + /// A request [`Self::try_acquire`] refuses names at least one. + pub fn contention(&self, txn: TxnId, keys: &BTreeMap) -> Vec { + keys.iter() + .filter(|(key, mode)| self.classify(txn, key, **mode) == KeyState::Conflict) + .filter_map(|(key, mode)| { + let entry = self.table.get(key)?; + Some(KeyContention { + key: key.clone(), + requested: *mode, + held: entry.mode, + holders: entry.holders.to_vec(), + waiters: entry.waiters.iter().copied().collect(), + }) + }) + .collect() + } + /// Check whether a previously-blocked transaction is now ready. /// /// A transaction is ready when for every key in its key set, the key is @@ -81,6 +112,28 @@ mod tests { TxnId::new(epoch, pos) } + /// A refused request names each key that blocks it, with the holders + /// and the waiters ahead. A free key names nothing. + #[test] + fn contention_names_the_keys_that_block_a_request() { + let mut lm = LockManager::new(); + let (holder, waiter, probe) = (txn(1, 0), txn(1, 1), txn(1, 2)); + lm.acquire(holder, keyset(&["x"])); + lm.acquire(waiter, keyset(&["x"])); + let request = keyset(&["x", "free"]); + assert!(!lm.try_acquire(probe, request.clone())); + assert_eq!( + lm.contention(probe, &request), + vec![KeyContention { + key: key("x"), + requested: LockMode::Exclusive, + held: LockMode::Exclusive, + holders: vec![holder], + waiters: vec![(waiter, LockMode::Exclusive)], + }] + ); + } + #[test] fn is_ready_returns_true_when_all_keys_free_or_self_at_front() { let mut lm = LockManager::new(); diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs index 79746d7be..619b57359 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/manager/mod.rs @@ -27,7 +27,8 @@ //! resolution. //! - [`release`]: lock release and mode-aware FIFO waiter promotion. //! - [`try_acquire`]: the non-blocking fast path. -//! - [`introspection`]: readiness checks and test counters. +//! - [`introspection`]: readiness checks, request contention and test +//! counters. mod acquire; mod classify; @@ -37,4 +38,5 @@ mod try_acquire; mod types; mod wound_wait; +pub use introspection::KeyContention; pub use types::LockManager; diff --git a/nodedb/src/control/cluster/calvin/scheduler/lock/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/lock/mod.rs index 8f00d1d06..964bdeb07 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/lock/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/lock/mod.rs @@ -9,4 +9,4 @@ pub mod reap; pub use hot_key_table::HotKeyTable; pub use lock_entry::{AcquireOutcome, LockMode}; pub use lock_key::{LockKey, TxnId}; -pub use manager::LockManager; +pub use manager::{KeyContention, LockManager}; diff --git a/nodedb/src/control/cluster/calvin/scheduler/mod.rs b/nodedb/src/control/cluster/calvin/scheduler/mod.rs index 290645f11..f9d862066 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/mod.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/mod.rs @@ -2,6 +2,8 @@ pub mod applied_gate; pub mod applied_ledger; +pub mod barrier_log; +pub mod barrier_store; pub mod caught_up; pub mod cut_floor; pub mod cut_hold; @@ -10,14 +12,16 @@ pub mod inbox; pub mod lock; pub mod metrics; mod metrics_flow; +pub mod read_results; pub mod recovery; pub use applied_gate::AppliedGate; pub use applied_ledger::{CalvinAppliedLedger, CalvinAppliedLedgers, ClaimRefusal}; +pub use barrier_log::{BarrierEvent, BarrierLog, BarrierOutcome, ReceivedReads}; pub use caught_up::{CaughtUpHandle, CaughtUpRegistry}; pub use driver::{ - CalvinReadResultProposal, RaftSequencerProposer, ReadResultEvent, Scheduler, SchedulerConfig, - SchedulerParams, SequencerProposer, propose_calvin_read_result, + CalvinReadResultProposal, RaftSequencerProposer, Scheduler, SchedulerConfig, SchedulerParams, + SequencerProposer, propose_calvin_read_result, }; pub use inbox::{CalvinApplyEvent, CalvinInboxes, CalvinVShardInbox, InboxHandle}; pub use lock::{AcquireOutcome, HotKeyTable, LockKey, LockManager, LockMode, TxnId}; @@ -25,4 +29,5 @@ pub use lock::{AcquireOutcome, HotKeyTable, LockKey, LockManager, LockMode, TxnI // keep that path stable via an alias while the module lives under `lock/`. pub use lock as lock_manager; pub use metrics::SchedulerMetrics; +pub use read_results::{BufferedEvents, CalvinReadResults, VShardReadResults}; pub use recovery::{AppliedRecovery, NOT_YET_APPLIED_EPOCH, recover_all_applied}; diff --git a/nodedb/src/control/cluster/calvin/scheduler/read_results.rs b/nodedb/src/control/cluster/calvin/scheduler/read_results.rs new file mode 100644 index 000000000..3815deef9 --- /dev/null +++ b/nodedb/src/control/cluster/calvin/scheduler/read_results.rs @@ -0,0 +1,511 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Dependent-read barrier events that reached a vShard before its scheduler +//! took them. +//! +//! The data-group apply loop folds every `CalvinReadResult` and +//! `CalvinReadTimeout` entry into the txn's stored barrier row (see +//! [`super::barrier_store`]), then into the vShard's [`VShardReadResults`]. +//! The entry can apply before this node's scheduler granted the txn, or +//! before a scheduler for the vShard runs here at all. The events then wait +//! here. The scheduler takes a txn's events once it holds the txn: at the +//! grant, and on each push for a txn it holds. +//! +//! A txn's events wait in one of two places: +//! +//! - in memory, as a folded [`BarrierLog`]; +//! - in its stored row alone. Such a txn is "stored", and the scheduler reads +//! the row when it takes the txn. +//! +//! Memory holds at most `capacity` events per vShard. An event past it moves +//! its txn to stored: the row holds every saved event of the txn, so nothing +//! is dropped. Boot and a snapshot install mark the txns of the rows they +//! find as stored. +//! +//! An event whose row write failed stays in memory, past the bound. Its txn +//! becomes "unsaved": the row then holds a prefix of the txn's events, so +//! every later event of the txn stays in memory too and skips the row. The +//! row stays a prefix, so a restart that delivers the failed entry again +//! folds the events in log order. + +use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +use tokio::sync::Notify; + +use super::barrier_log::{BarrierEvent, BarrierLog}; +use super::lock::TxnId; + +/// The waiting events of one txn, as [`VShardReadResults::take`] hands +/// them out. +#[derive(Debug, Default, PartialEq)] +pub struct BufferedEvents { + /// The events memory held, folded. + pub memory: BarrierLog, + /// Whether the txn's stored row holds events memory does not. Its + /// events are the row's, extended with `memory`. + pub stored: bool, +} + +#[derive(Debug, Default)] +struct Buffered { + logs: BTreeMap, + /// Sum of `event_count` over `logs`. + events: usize, + /// Txns whose stored row holds events memory does not. + stored: BTreeSet, + /// Txns with an event the stored row lacks. + unsaved: BTreeSet, +} + +impl Buffered { + fn remove_log(&mut self, txn: TxnId) -> Option { + let log = self.logs.remove(&txn)?; + self.events = self.events.saturating_sub(log.event_count()); + Some(log) + } + + /// Fold `log`, events after every event memory holds for `txn`. + fn extend_log(&mut self, txn: TxnId, later: BarrierLog) { + let mut log = self.remove_log(txn).unwrap_or_default(); + log.extend(later); + self.events += log.event_count(); + self.logs.insert(txn, log); + } + + /// Take `txn`'s waiting events, when it has any. + fn take(&mut self, txn: TxnId) -> Option { + let memory = self.remove_log(txn); + let stored = self.stored.remove(&txn); + if memory.is_none() && !stored { + return None; + } + Some(BufferedEvents { + memory: memory.unwrap_or_default(), + stored, + }) + } + + /// Whether `txn` has waiting events. + fn waits(&self, txn: TxnId) -> bool { + self.logs.contains_key(&txn) || self.stored.contains(&txn) + } +} + +/// One vShard's waiting barrier events. +#[derive(Debug)] +pub struct VShardReadResults { + vshard_id: u32, + state: Mutex, + /// Wakes the scheduler: an event arrived. + pushed: Notify, + /// Read-result entries the apply loop applied for the vShard, a finished + /// txn's included. + reads_applied: AtomicU64, + /// The `(group, log index)` of the latest read-result entries applied, + /// oldest first, at most [`READ_ENTRY_HISTORY`]. + read_entries: Mutex>, +} + +/// How many applied read-result entries a vShard's buffer names. +pub const READ_ENTRY_HISTORY: usize = 64; + +impl VShardReadResults { + fn new(vshard_id: u32) -> Self { + Self { + vshard_id, + state: Mutex::new(Buffered::default()), + pushed: Notify::new(), + reads_applied: AtomicU64::new(0), + read_entries: Mutex::new(VecDeque::new()), + } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Buffered> { + self.state.lock().unwrap_or_else(|p| p.into_inner()) + } + + /// The vShard this buffer serves. + pub fn vshard_id(&self) -> u32 { + self.vshard_id + } + + /// Count one read-result entry the apply loop applied, at `log_index` + /// of `group_id`. + pub fn note_read_applied(&self, group_id: u64, log_index: u64) { + self.reads_applied.fetch_add(1, Ordering::Relaxed); + let mut entries = self.read_entries.lock().unwrap_or_else(|p| p.into_inner()); + if entries.len() == READ_ENTRY_HISTORY { + entries.pop_front(); + } + entries.push_back((group_id, log_index)); + } + + /// The `(group, log index)` of the latest read-result entries applied + /// for the vShard since this buffer was created, oldest first. + pub fn read_entries(&self) -> Vec<(u64, u64)> { + self.read_entries + .lock() + .unwrap_or_else(|p| p.into_inner()) + .iter() + .copied() + .collect() + } + + /// How many read-result entries the apply loop applied for the vShard + /// since this buffer was created. + pub fn reads_applied(&self) -> u64 { + self.reads_applied.load(Ordering::Relaxed) + } + + /// Whether an event of `txn` is missing from its stored row. Every later + /// event of `txn` then skips the row. + pub fn is_unsaved(&self, txn: TxnId) -> bool { + self.lock().unsaved.contains(&txn) + } + + /// Whether some txn of the vShard has an event its stored row lacks. + pub fn has_unsaved(&self) -> bool { + !self.lock().unsaved.is_empty() + } + + /// Fold `event` of `txn`. `saved` says the event's row write + /// succeeded. + /// + /// A saved event of a stored txn stays in its row alone. A saved event + /// past `capacity` moves its txn to stored. An unsaved event stays in + /// memory whatever the bound, and marks its txn unsaved. + pub fn push(&self, txn: TxnId, event: BarrierEvent, capacity: usize, saved: bool) { + { + let mut state = self.lock(); + let in_row_alone = saved && !state.unsaved.contains(&txn); + if !in_row_alone { + state.unsaved.insert(txn); + } + let skip = if !in_row_alone { + false + } else if state.stored.contains(&txn) { + true + } else { + let grows = state.logs.get(&txn).is_none_or(|log| log.accepts(&event)); + if grows && state.events >= capacity { + state.remove_log(txn); + state.stored.insert(txn); + true + } else { + false + } + }; + if !skip { + let mut later = BarrierLog::default(); + later.note(event); + state.extend_log(txn, later); + } + } + self.pushed.notify_one(); + } + + /// Take `txn`'s waiting events. + pub fn take(&self, txn: TxnId) -> Option { + self.lock().take(txn) + } + + /// Take the waiting events of every txn `held` names, in txn order. + pub fn take_held(&self, held: impl Fn(TxnId) -> bool) -> Vec<(TxnId, BufferedEvents)> { + let mut state = self.lock(); + let txns: BTreeSet = state + .logs + .keys() + .chain(state.stored.iter()) + .copied() + .filter(|txn| held(*txn)) + .collect(); + txns.into_iter() + .filter_map(|txn| state.take(txn).map(|events| (txn, events))) + .collect() + } + + /// Hand back `earlier`, the events of `txn` a scheduler took and did + /// not finish. They come before any event buffered since. `stored` + /// says the txn's row holds events `earlier` lacks. A scheduler holds + /// at most its in-flight backlog of txns, so the hand back takes no + /// capacity check, and the next take returns them. It wakes no + /// scheduler: the events are ones a scheduler took already. + pub fn restore(&self, txn: TxnId, earlier: BufferedEvents) { + let mut state = self.lock(); + let later = state.remove_log(txn); + let mut log = earlier.memory; + if let Some(later) = later { + log.extend(later); + } + state.extend_log(txn, log); + if earlier.stored { + state.stored.insert(txn); + } + } + + /// Mark `txns` stored: their rows hold their events. + pub fn mark_stored(&self, txns: impl IntoIterator) { + self.lock().stored.extend(txns); + self.pushed.notify_one(); + } + + /// Replace every waiting event with the rows of `stored`. A snapshot + /// install replaced the vShard's rows with the builder's. + pub fn reset_to_stored(&self, stored: BTreeSet) { + { + let mut state = self.lock(); + *state = Buffered { + stored, + ..Buffered::default() + }; + } + self.pushed.notify_one(); + } + + /// Drop every waiting event of `txn`, which finished. Returns whether + /// it had any. + pub fn forget_txn(&self, txn: TxnId) -> bool { + let mut state = self.lock(); + let unsaved = state.unsaved.remove(&txn); + state.take(txn).is_some() || unsaved + } + + /// Drop the events of every txn `finished` names: its position is + /// applied, so no barrier of it opens again. + pub fn discard_finished(&self, finished: impl Fn(TxnId) -> bool) { + let mut state = self.lock(); + let done: Vec = state + .logs + .keys() + .chain(state.stored.iter()) + .chain(state.unsaved.iter()) + .copied() + .filter(|txn| finished(*txn)) + .collect(); + for txn in done { + state.take(txn); + state.unsaved.remove(&txn); + } + } + + /// Resolves once an event was pushed since the last wake. + pub async fn pushed(&self) { + self.pushed.notified().await; + } + + /// How many txns have waiting events, in memory or in their row. + pub fn waiting_txns(&self) -> usize { + let state = self.lock(); + state + .logs + .keys() + .filter(|txn| !state.stored.contains(txn)) + .count() + + state.stored.len() + } + + /// Whether `txn` has waiting events. + pub fn waits(&self, txn: TxnId) -> bool { + self.lock().waits(txn) + } +} + +/// The waiting barrier events of every vShard of this node. +#[derive(Debug, Default)] +pub struct CalvinReadResults { + by_vshard: Mutex>>, +} + +impl CalvinReadResults { + /// The buffer of `vshard_id`, created empty on first use. + pub fn vshard(&self, vshard_id: u32) -> Arc { + let mut by_vshard = self.by_vshard.lock().unwrap_or_else(|p| p.into_inner()); + Arc::clone( + by_vshard + .entry(vshard_id) + .or_insert_with(|| Arc::new(VShardReadResults::new(vshard_id))), + ) + } + + /// How many txns of `vshard_id` have waiting events. + pub fn waiting_txns(&self, vshard_id: u32) -> usize { + self.by_vshard + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&vshard_id) + .map_or(0, |buffer| buffer.waiting_txns()) + } + + /// Drop the buffer of `vshard_id`, which left this node. + pub fn forget(&self, vshard_id: u32) { + self.by_vshard + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(&vshard_id); + } +} + +#[cfg(test)] +mod tests { + use nodedb_physical::physical_plan::meta::PassiveReadKeyId; + use nodedb_types::{QualifiedCollection, Value}; + + use super::*; + + fn read(passive_vshard: u32) -> BarrierEvent { + BarrierEvent::Read { + passive_vshard, + values: vec![( + PassiveReadKeyId::surrogate(QualifiedCollection::from_stored("c".to_owned()), 1), + Value::Null, + )], + } + } + + /// Events of one txn fold into one log, in push order. + #[test] + fn events_of_a_txn_fold_in_push_order() { + let buffer = VShardReadResults::new(4); + let txn = TxnId::new(3, 1); + buffer.push(txn, read(1), 8, true); + buffer.push(txn, BarrierEvent::Timeout, 8, true); + buffer.push(txn, read(2), 8, true); + assert_eq!(buffer.waiting_txns(), 1); + let events = buffer.take(txn).expect("buffered log"); + assert!(!events.stored); + assert_eq!( + events.memory.missing(&[1, 2].into_iter().collect()), + vec![2] + ); + assert_eq!(buffer.waiting_txns(), 0); + } + + /// A saved event past the bound moves its txn to its row, and every + /// later saved event of it stays in the row alone. + #[test] + fn a_saved_event_past_the_bound_moves_its_txn_to_its_row() { + let buffer = VShardReadResults::new(4); + let first = TxnId::new(1, 0); + let second = TxnId::new(1, 1); + buffer.push(first, read(1), 1, true); + buffer.push(second, read(1), 1, true); + buffer.push(second, BarrierEvent::Timeout, 1, true); + assert_eq!(buffer.waiting_txns(), 2); + // A duplicate of a held event takes no room and moves nothing. + buffer.push(first, read(1), 1, true); + let first_events = buffer.take(first).expect("first txn"); + assert!(!first_events.stored); + assert_eq!(first_events.memory.event_count(), 1); + let second_events = buffer.take(second).expect("second txn"); + assert_eq!( + second_events, + BufferedEvents { + memory: BarrierLog::default(), + stored: true, + } + ); + } + + /// An unsaved event stays in memory past the bound, and every later + /// event of its txn stays in memory too. + #[test] + fn an_unsaved_event_stays_in_memory_with_every_later_one() { + let buffer = VShardReadResults::new(4); + let full = TxnId::new(1, 0); + let txn = TxnId::new(2, 0); + buffer.push(full, read(1), 1, true); + buffer.push(txn, read(1), 1, false); + assert!(buffer.is_unsaved(txn)); + assert!(buffer.has_unsaved()); + buffer.push(txn, BarrierEvent::Timeout, 1, true); + let events = buffer.take(txn).expect("unsaved txn"); + assert!(!events.stored); + assert_eq!(events.memory.event_count(), 2); + assert!(buffer.is_unsaved(txn), "the row still lacks the event"); + assert!(buffer.forget_txn(txn)); + assert!(!buffer.has_unsaved()); + } + + /// Only the txns the scheduler holds leave the buffer, stored ones too. + #[test] + fn take_held_takes_only_held_txns() { + let buffer = VShardReadResults::new(4); + buffer.push(TxnId::new(1, 0), read(1), 8, true); + buffer.push(TxnId::new(2, 0), read(1), 8, true); + buffer.mark_stored([TxnId::new(3, 0)]); + let taken = buffer.take_held(|txn| txn.epoch >= 2); + let txns: Vec = taken.iter().map(|(txn, _)| *txn).collect(); + assert_eq!(txns, vec![TxnId::new(2, 0), TxnId::new(3, 0)]); + assert!(taken[1].1.stored); + assert_eq!(buffer.waiting_txns(), 1); + } + + /// Events a stopping scheduler hands back come before the ones buffered + /// since, and a finished txn's events leave the buffer. + #[test] + fn restore_keeps_log_order_and_discard_drops_finished() { + let buffer = VShardReadResults::new(4); + let txn = TxnId::new(5, 0); + buffer.push(txn, BarrierEvent::Timeout, 8, true); + let mut earlier = BarrierLog::default(); + earlier.note(read(1)); + buffer.restore( + txn, + BufferedEvents { + memory: earlier, + stored: true, + }, + ); + let events = buffer.take(txn).expect("restored log"); + assert!(events.stored); + assert!(events.memory.missing(&[1].into_iter().collect()).is_empty()); + + buffer.push(TxnId::new(6, 0), read(1), 8, false); + buffer.mark_stored([TxnId::new(7, 0)]); + buffer.discard_finished(|txn| txn.epoch >= 6); + assert_eq!(buffer.waiting_txns(), 0); + assert!(!buffer.has_unsaved()); + } + + /// A snapshot install replaces every waiting event with its rows. + #[test] + fn a_reset_keeps_only_the_installed_rows() { + let buffer = VShardReadResults::new(4); + buffer.push(TxnId::new(1, 0), read(1), 8, false); + buffer.reset_to_stored(BTreeSet::from([TxnId::new(2, 0)])); + assert!(!buffer.has_unsaved()); + assert!(!buffer.waits(TxnId::new(1, 0))); + assert!(buffer.waits(TxnId::new(2, 0))); + } + + /// The buffer names the latest applied read-result entries, oldest + /// first, and drops the oldest past the history bound. + #[test] + fn applied_read_entries_keep_the_latest() { + let buffer = VShardReadResults::new(4); + for index in 0..(READ_ENTRY_HISTORY as u64 + 2) { + buffer.note_read_applied(3, index); + } + let entries = buffer.read_entries(); + assert_eq!(entries.len(), READ_ENTRY_HISTORY); + assert_eq!(entries.first(), Some(&(3, 2))); + assert_eq!(buffer.reads_applied(), READ_ENTRY_HISTORY as u64 + 2); + } + + /// A push wakes a waiting scheduler. + #[tokio::test] + async fn a_push_wakes_the_scheduler() { + let buffer = Arc::new(VShardReadResults::new(4)); + let waiter = { + let buffer = Arc::clone(&buffer); + tokio::spawn(async move { buffer.pushed().await }) + }; + buffer.push(TxnId::new(1, 0), read(1), 8, true); + tokio::time::timeout(std::time::Duration::from_secs(5), waiter) + .await + .expect("the push wakes the waiter") + .expect("waiter task"); + } +} diff --git a/nodedb/src/control/cluster/calvin_snapshot/capture.rs b/nodedb/src/control/cluster/calvin_snapshot/capture.rs index 36dc18404..cc9fb8fea 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/capture.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/capture.rs @@ -17,6 +17,7 @@ use std::time::Duration; use nodedb_cluster::calvin::SequencerEntry; use crate::Error; +use crate::control::cluster::calvin::scheduler::barrier_store; use crate::control::security::catalog::calvin_base::CalvinBase; use crate::control::state::SharedState; use crate::types::{GroupCalvinCut, VShardCalvinState}; @@ -103,19 +104,23 @@ pub async fn await_calvin_cut( } /// The group's Calvin cut through `through`: the applied positions of every -/// group vShard here. The caller holds the group's apply gate exclusive, so -/// no Calvin install of these vShards runs. +/// group vShard here, and the stored barrier logs of their unfinished txns. +/// The caller holds the group's apply gate exclusive, so no Calvin install +/// of these vShards runs and no barrier entry of the group applies. +/// +/// Fails when the barrier logs cannot be read whole (see +/// [`barrier_store::capture_group`]). The build retries on the next +/// heartbeat. pub fn capture_calvin_cut( shared: &SharedState, group_vshards: &HashSet, through: u64, -) -> GroupCalvinCut { +) -> Result { let ledgers = &shared.calvin.applied; - let mut ids: Vec = group_vshards.iter().copied().collect(); - ids.sort_unstable(); + let ids: BTreeSet = group_vshards.iter().copied().collect(); let vshards = ids - .into_iter() - .filter_map(|vshard_id| { + .iter() + .filter_map(|&vshard_id| { let (fully_applied_epoch, tail) = ledgers.get(vshard_id)?.snapshot(); Some(VShardCalvinState { vshard_id, @@ -124,5 +129,10 @@ pub fn capture_calvin_cut( }) }) .collect(); - GroupCalvinCut { through, vshards } + let barrier_logs = barrier_store::capture_group(shared, &ids)?; + Ok(GroupCalvinCut { + through, + vshards, + barrier_logs, + }) } diff --git a/nodedb/src/control/cluster/calvin_snapshot/install.rs b/nodedb/src/control/cluster/calvin_snapshot/install.rs index 878420faf..582ec801d 100644 --- a/nodedb/src/control/cluster/calvin_snapshot/install.rs +++ b/nodedb/src/control/cluster/calvin_snapshot/install.rs @@ -11,11 +11,15 @@ //! the install installs nothing more, and the next scheduler reconcile //! starts it again from the installed state. //! +//! The stored dependent-read barrier logs of the group's vShards become the +//! cut's too: the snapshot covers barrier entries this node never applies. +//! //! A WAL `SnapshotInstalled` record of the group, written before this runs, //! makes boot recovery drop the vShards' applied markers from before it. use std::collections::{BTreeSet, HashSet}; +use crate::control::cluster::calvin::scheduler::barrier_store; use crate::control::cluster::snapshot_install::SnapshotInstallError; use crate::control::security::catalog::calvin_applied::StoredCalvinApplied; use crate::control::state::SharedState; @@ -68,6 +72,10 @@ pub fn install_calvin_cut( catalog .replace_calvin_applied(&states) .map_err(settle_error)?; + // The receiver never applies the barrier entries the snapshot covers: + // the builder's stored logs of the unfinished txns replace this node's. + let vshards: BTreeSet = ids.iter().copied().collect(); + barrier_store::install_group(shared, &vshards, cut.barrier_logs).map_err(settle_error)?; shared .calvin .bases diff --git a/nodedb/src/control/cluster/snapshot_builder.rs b/nodedb/src/control/cluster/snapshot_builder.rs index 050590b8f..c67b429ca 100644 --- a/nodedb/src/control/cluster/snapshot_builder.rs +++ b/nodedb/src/control/cluster/snapshot_builder.rs @@ -358,7 +358,8 @@ impl nodedb_cluster::SnapshotBuilder for DataPlaneSnapshotBuilder { .map_err(|e| Box::new(e) as Box)?; // Read under the fence: the storage captured below holds exactly the // Calvin positions this names. - let calvin_cut = capture_calvin_cut(&self.shared, &group_vshards, calvin_through); + let calvin_cut = capture_calvin_cut(&self.shared, &group_vshards, calvin_through) + .map_err(|e| Box::new(e) as Box)?; // Enumerate tenants and their databases from the system catalog — the // same source the backup orchestrator's catalog sections use. Every diff --git a/nodedb/src/control/cluster/start_raft/core.rs b/nodedb/src/control/cluster/start_raft/core.rs index 08bd79e3e..1628977d6 100644 --- a/nodedb/src/control/cluster/start_raft/core.rs +++ b/nodedb/src/control/cluster/start_raft/core.rs @@ -66,7 +66,6 @@ pub async fn start_raft( &loop_build.raft_loop, loop_build.tracker, loop_build.apply_rx, - loop_build.calvin_read_result_senders, loop_build.sequencer_state_machine, )?; // The async proposer is installed: a failed redo stream abandon can be diff --git a/nodedb/src/control/cluster/start_raft/group_setup.rs b/nodedb/src/control/cluster/start_raft/group_setup.rs index bdf8aebdf..389644a16 100644 --- a/nodedb/src/control/cluster/start_raft/group_setup.rs +++ b/nodedb/src/control/cluster/start_raft/group_setup.rs @@ -6,18 +6,16 @@ //! snapshot-transfer / replication-factor config that must be read before //! `pending_subsystems` is consumed. -use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; use std::time::Duration; -use tokio::sync::mpsc::{self, Sender}; +use tokio::sync::mpsc; use nodedb_cluster::calvin::{ CalvinCompletionRegistry, SEQUENCER_GROUP_ID, SequencerStateMachine, TxnId, VerdictOutcome, }; use crate::control::cluster::array_executor::DataPlaneArrayExecutor; -use crate::control::cluster::calvin::ReadResultEvent; use crate::control::cluster::handle::ClusterHandle; use crate::control::cluster::metadata_applier::{MetadataCommitApplier, seed_metadata_cache}; use crate::control::cluster::spsc_applier::SpscCommitApplier; @@ -38,7 +36,6 @@ pub(super) struct GroupSetup { /// vote tally completes. pub(super) calvin_verdict_rx: mpsc::Receiver<(TxnId, VerdictOutcome)>, pub(super) sequencer_state_machine: Arc>, - pub(super) calvin_read_result_senders: Arc>>>, pub(super) metadata_applier: Arc, pub(super) token_state: nodedb_cluster::SharedTokenStateMirror, pub(super) plan_executor: Arc, @@ -110,7 +107,6 @@ pub(super) fn build_group_setup( calvin_completion_registry: calvin.completion_registry, calvin_verdict_rx: calvin.verdict_rx, sequencer_state_machine: calvin.sequencer_state_machine, - calvin_read_result_senders: calvin.read_result_senders, metadata_applier, token_state, plan_executor, @@ -165,7 +161,6 @@ struct CalvinState { completion_registry: Arc, verdict_rx: mpsc::Receiver<(TxnId, VerdictOutcome)>, sequencer_state_machine: Arc>, - read_result_senders: Arc>>>, } /// Build the Calvin completion registry, its verdict channel, and the @@ -238,12 +233,10 @@ fn build_calvin_state(handle: &ClusterHandle, shared: &Arc) -> Calv shared, ))), )); - let read_result_senders = Arc::new(Mutex::new(BTreeMap::>::new())); CalvinState { completion_registry: calvin_completion_registry, verdict_rx: calvin_verdict_rx, sequencer_state_machine, - read_result_senders, } } diff --git a/nodedb/src/control/cluster/start_raft/hooks.rs b/nodedb/src/control/cluster/start_raft/hooks.rs index 19ee735a9..a68f13243 100644 --- a/nodedb/src/control/cluster/start_raft/hooks.rs +++ b/nodedb/src/control/cluster/start_raft/hooks.rs @@ -114,6 +114,13 @@ pub(super) async fn build_hooks( .is_some_and(|ledger| ledger.is_applied(epoch, position)) }); + // Every stored dependent-read barrier log of an unfinished txn waits in + // its vShard's buffer, and the rows of finished txns go. A barrier entry + // at or below a group's durable applied floor is never delivered again, + // so its row is what a barrier reads from now on. A boot completion of a + // snapshot install below replaces its group's rows. + crate::control::cluster::calvin::scheduler::barrier_store::restore_at_boot(shared)?; + // Per-group snapshot builder for the SEND path: on the leader, build the // real serialized engine state for a lagging follower's group vshards // (replacing the prior empty stub bytes). diff --git a/nodedb/src/control/cluster/start_raft/loop_build.rs b/nodedb/src/control/cluster/start_raft/loop_build.rs index e51fc48b1..527aca8ab 100644 --- a/nodedb/src/control/cluster/start_raft/loop_build.rs +++ b/nodedb/src/control/cluster/start_raft/loop_build.rs @@ -5,19 +5,18 @@ //! service, spawn the vShard schedulers, and start the cluster subsystems //! (health/gossip/etc.) that share the loop's `MultiRaft`. -use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; -use tokio::sync::mpsc::{self, Sender}; +use tokio::sync::mpsc; use nodedb_cluster::calvin::{ CalvinCompletionRegistry, SequencerConfig, SequencerReceivers, SequencerService, SequencerStateMachine, new_inbox, new_reservation_inbox, }; +use crate::control::cluster::calvin::SchedulerConfig; use crate::control::cluster::calvin::executor::ollp::OllpConfig; use crate::control::cluster::calvin::executor::ollp::orchestrator::OllpOrchestrator; -use crate::control::cluster::calvin::{ReadResultEvent, SchedulerConfig}; use crate::control::cluster::handle::ClusterHandle; use crate::control::cluster::start_raft_helpers::{ SpawnVshardSchedulersParams, spawn_vshard_schedulers, @@ -47,7 +46,6 @@ pub(super) struct LoopBuild { pub(super) ollp_orchestrator: Arc, pub(super) tracker: Arc, pub(super) apply_rx: mpsc::Receiver, - pub(super) calvin_read_result_senders: Arc>>>, pub(super) calvin_completion_registry: Arc, pub(super) token_state: nodedb_cluster::SharedTokenStateMirror, /// Shared with the compactor wiring so sequencer-group log compaction can @@ -78,7 +76,6 @@ pub(super) async fn build_raft_loop( calvin_completion_registry, calvin_verdict_rx, sequencer_state_machine, - calvin_read_result_senders, metadata_applier, token_state, plan_executor, @@ -216,7 +213,6 @@ pub(super) async fn build_raft_loop( shared, raft_loop_handle: raft_loop_handle.clone(), sequencer_state_machine: &sequencer_state_machine, - calvin_read_result_senders: &calvin_read_result_senders, calvin_completion_registry: &calvin_completion_registry, scheduler_config: &scheduler_config, })?; @@ -266,7 +262,6 @@ pub(super) async fn build_raft_loop( ollp_orchestrator, tracker, apply_rx, - calvin_read_result_senders, calvin_completion_registry, token_state, sequencer_state_machine, diff --git a/nodedb/src/control/cluster/start_raft/propose_error.rs b/nodedb/src/control/cluster/start_raft/propose_error.rs index 7f73da055..54001390e 100644 --- a/nodedb/src/control/cluster/start_raft/propose_error.rs +++ b/nodedb/src/control/cluster/start_raft/propose_error.rs @@ -46,7 +46,12 @@ pub(super) fn async_propose_error(vshard_id: u32, error: ClusterError) -> crate: // deadline class, the same class the array fan-out gives it. A // forward whose stream failed after the request was written can // also have been proposed, so it takes the same class. - ClusterError::ShardTimeout { .. } | ClusterError::Unanswered { .. } => { + error @ (ClusterError::ShardTimeout { .. } | ClusterError::Unanswered { .. }) => { + tracing::warn!( + vshard_id, + %error, + "raft proposal forward reached its deadline with no answer from the leader" + ); crate::Error::DeadlineExceeded { request_id: crate::types::RequestId::new(0), } @@ -54,7 +59,13 @@ pub(super) fn async_propose_error(vshard_id: u32, error: ClusterError) -> crate: ClusterError::DataPlane { code } => crate::Error::DataPlane(code.into()), // The leader's write gate waited for the write's lock keys until the // caller's deadline, and proposed nothing. - ClusterError::Calvin(CalvinError::AdmissionTimedOut) => expired_before_propose(), + ClusterError::Calvin(CalvinError::AdmissionTimedOut) => { + tracing::warn!( + vshard_id, + "raft proposal reached its deadline at the leader's write gate" + ); + expired_before_propose() + } ClusterError::ShardExecution { error, .. } | ClusterError::StreamTerminal { error, .. } => { crate::Error::from(*error) } diff --git a/nodedb/src/control/cluster/start_raft/proposer_wiring.rs b/nodedb/src/control/cluster/start_raft/proposer_wiring.rs index f83b79065..264ae0d08 100644 --- a/nodedb/src/control/cluster/start_raft/proposer_wiring.rs +++ b/nodedb/src/control/cluster/start_raft/proposer_wiring.rs @@ -5,12 +5,10 @@ //! background apply loop that drains `DistributedApplier::apply_committed` //! into the Data Plane and notifies propose waiters. -use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; -use tokio::sync::mpsc::{self, Sender}; +use tokio::sync::mpsc; -use crate::control::cluster::calvin::ReadResultEvent; use crate::control::distributed_applier::{ApplyBatch, ProposeTracker, run_apply_loop}; use crate::control::state::SharedState; @@ -25,7 +23,6 @@ pub(super) fn wire_proposers( raft_loop: &Arc, tracker: Arc, apply_rx: mpsc::Receiver, - calvin_read_result_senders: Arc>>>, sequencer_state_machine: Arc>, ) -> crate::Result<()> { install_sync_proposer(shared, raft_loop); @@ -33,7 +30,7 @@ pub(super) fn wire_proposers( install_applied_index_sink(shared, raft_loop); install_apply_gates(shared, raft_loop); install_async_proposer(shared, raft_loop, &tracker)?; - spawn_apply_loop(shared, tracker, apply_rx, calvin_read_result_senders); + spawn_apply_loop(shared, tracker, apply_rx); Ok(()) } @@ -332,7 +329,6 @@ fn spawn_apply_loop( shared: &Arc, tracker: Arc, apply_rx: mpsc::Receiver, - calvin_read_result_senders: Arc>>>, ) { // Spawn the background apply loop. It reads from the mpsc channel // pushed by `DistributedApplier::apply_committed`, dispatches to the @@ -344,7 +340,6 @@ fn spawn_apply_loop( // because its applies dispatch to the Data Plane. let apply_state = shared.clone(); let apply_tracker = tracker; - let apply_calvin_read_result_senders = calvin_read_result_senders; crate::control::shutdown::spawn_loop_no_abort( &shared.loop_registry, &shared.shutdown, @@ -357,12 +352,7 @@ fn spawn_apply_loop( // cuts the loop off mid-apply. tokio::select! { biased; - _ = run_apply_loop( - apply_rx, - apply_state, - apply_tracker, - apply_calvin_read_result_senders, - ) => {} + _ = run_apply_loop(apply_rx, apply_state, apply_tracker) => {} _ = shutdown.wait_cancelled() => {} } }, @@ -469,6 +459,7 @@ async fn await_local_apply( tracing::warn!( group_id, log_index, + vshard_id, progress = %apply_progress(state.as_deref(), group_id), oldest_unfinished = %tracker .applying(group_id) diff --git a/nodedb/src/control/cluster/start_raft_helpers.rs b/nodedb/src/control/cluster/start_raft_helpers.rs index 1642ad138..04be91339 100644 --- a/nodedb/src/control/cluster/start_raft_helpers.rs +++ b/nodedb/src/control/cluster/start_raft_helpers.rs @@ -1,21 +1,20 @@ // SPDX-License-Identifier: BUSL-1.1 +use std::collections::BTreeSet; use std::sync::{Arc, Mutex, RwLock}; use nodedb_cluster::calvin::{CalvinCompletionRegistry, SequencerStateMachine}; use crate::control::cluster::calvin::scheduler::metrics::SchedulerMetrics; use crate::control::cluster::calvin::{ - RaftSequencerProposer, ReadResultEvent, Scheduler, SchedulerConfig, SchedulerParams, - SequencerProposer, + RaftSequencerProposer, Scheduler, SchedulerConfig, SchedulerParams, SequencerProposer, }; use crate::control::cluster::calvin_snapshot; use crate::control::cluster::handle::ClusterHandle; use crate::control::state::SharedState; -/// Type alias for the shared per-vShard read-result sender registry. -type ReadResultSenders = - Arc>>>; +/// The vShards whose scheduler this node started and has not stopped. +type ServedVShards = Arc>>; /// The vShards this node currently hosts: the union of `vshards_for_group` over /// every Raft group whose member set includes this node, read from the live @@ -41,20 +40,19 @@ fn hosted_vshards(routing: &RwLock, node_id: u64) /// them. A snapshot install replaced the state a running scheduler started /// from. Dropping the vShard's sequencer sender closes the scheduler's /// intake, and the scheduler exits. Its other channels and its lock table go -/// too. The next reconcile that finds the vShard hosted spawns a new +/// too. It hands the barrier events of the txns it did not finish back to +/// the vShard's read-result buffer. The next reconcile that finds the vShard hosted spawns a new /// scheduler from the vShard's base. fn retire_vshard_schedulers( hosted: &[u32], shared: &Arc, sequencer_state_machine: &Arc>, - calvin_read_result_senders: &ReadResultSenders, + served_vshards: &ServedVShards, calvin_completion_registry: &Arc, ) { let (left, rebased): (Vec, Vec) = { - let mut senders = calvin_read_result_senders - .lock() - .unwrap_or_else(|p| p.into_inner()); - let served: Vec = senders.keys().copied().collect(); + let mut served_set = served_vshards.lock().unwrap_or_else(|p| p.into_inner()); + let served: Vec = served_set.iter().copied().collect(); let left: Vec = served .iter() .copied() @@ -65,11 +63,28 @@ fn retire_vshard_schedulers( .filter(|vshard| !left.contains(vshard)) .collect(); for vshard in left.iter().chain(&rebased) { - senders.remove(vshard); + served_set.remove(vshard); } (left, rebased) }; calvin_snapshot::forget_left(shared, &left); + // A vShard that left this node keeps no barrier events here, in memory + // or stored. A rebased vShard keeps them for the scheduler that starts + // again. + for &vshard in &left { + if let Err(e) = + crate::control::cluster::calvin::scheduler::barrier_store::forget_vshard(shared, vshard) + { + // The rows stay. Boot marks them stored for a vShard with no + // scheduler, and a snapshot install of the vShard replaces them. + tracing::warn!( + vshard_id = vshard, + error = %e, + "calvin: the barrier rows of a vShard that left were not removed" + ); + crate::diag::calvin_barrier_log_store_failed(vshard, None, "remove", &e); + } + } for &vshard in &rebased { tracing::info!( vshard_id = vshard, @@ -122,7 +137,7 @@ struct ReconcileSchedulersParams<'a> { /// One proposer per node, so its forward limit bounds the whole node. sequencer_proposer: &'a Arc, sequencer_state_machine: &'a Arc>, - calvin_read_result_senders: &'a ReadResultSenders, + served_vshards: &'a ServedVShards, calvin_completion_registry: &'a Arc, scheduler_config: &'a SchedulerConfig, } @@ -130,10 +145,9 @@ struct ReconcileSchedulersParams<'a> { /// Idempotently ensure a Calvin `Scheduler` runs for exactly the vShards this /// node currently hosts. /// -/// A vShard is considered already-served iff it has a registered read-result -/// sender (the schedulers' presence registry). A served vShard this node no -/// longer hosts has its scheduler stopped first (see -/// [`retire_left_vshard_schedulers`]). Only newly-hosted vShards get a fresh +/// A vShard is already served when `served_vshards` holds it. A served +/// vShard this node no longer hosts has its scheduler stopped first (see +/// [`retire_vshard_schedulers`]). Only newly-hosted vShards get a fresh /// scheduler — this pass never double-spawns. Returns the number of NEW /// schedulers started. fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate::Result { @@ -144,7 +158,7 @@ fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate:: raft_loop_handle, sequencer_proposer, sequencer_state_machine, - calvin_read_result_senders, + served_vshards, calvin_completion_registry, scheduler_config, } = params; @@ -154,7 +168,7 @@ fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate:: &hosted, shared, sequencer_state_machine, - calvin_read_result_senders, + served_vshards, calvin_completion_registry, ); calvin_snapshot::retain_mounted(shared, raft_loop_handle, routing); @@ -166,10 +180,10 @@ fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate:: let mut kept = Vec::new(); for vshard_id in hosted { // Already-served vShards keep their running scheduler untouched. - if calvin_read_result_senders + if served_vshards .lock() .unwrap_or_else(|p| p.into_inner()) - .contains_key(&vshard_id) + .contains(&vshard_id) { continue; } @@ -232,12 +246,10 @@ fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate:: // The armed catch-up holds the log from here, so the base is kept. kept.push((vshard_id, first_available)); - let (read_result_tx, read_result_rx) = - tokio::sync::mpsc::channel(scheduler_config.channel_capacity); - calvin_read_result_senders + served_vshards .lock() .unwrap_or_else(|p| p.into_inner()) - .insert(vshard_id, read_result_tx); + .insert(vshard_id); // The deterministic lock table is shared between this scheduler and the // Control-Plane write-admission gate: build it once and register the @@ -289,7 +301,6 @@ fn reconcile_vshard_schedulers(params: ReconcileSchedulersParams<'_>) -> crate:: rebuild_target_epoch, config: scheduler_config.clone(), metrics: SchedulerMetrics::new(), - read_result_rx, lock_manager, promotion_rx, registry: Arc::clone(calvin_completion_registry), @@ -333,7 +344,6 @@ pub(super) struct SpawnVshardSchedulersParams<'a> { pub(super) shared: &'a Arc, pub(super) raft_loop_handle: Arc>, pub(super) sequencer_state_machine: &'a Arc>, - pub(super) calvin_read_result_senders: &'a ReadResultSenders, pub(super) calvin_completion_registry: &'a Arc, pub(super) scheduler_config: &'a SchedulerConfig, } @@ -364,12 +374,12 @@ pub(super) fn spawn_vshard_schedulers( shared, raft_loop_handle, sequencer_state_machine, - calvin_read_result_senders, calvin_completion_registry, scheduler_config, } = params; let node_id = handle.node_id; + let served_vshards: ServedVShards = Arc::new(Mutex::new(BTreeSet::new())); let routing = Arc::clone(&handle.routing); let sequencer_proposer: Arc = Arc::new(RaftSequencerProposer::new( node_id, @@ -394,7 +404,7 @@ pub(super) fn spawn_vshard_schedulers( raft_loop_handle: &raft_loop_handle, sequencer_proposer: &sequencer_proposer, sequencer_state_machine, - calvin_read_result_senders, + served_vshards: &served_vshards, calvin_completion_registry, scheduler_config, })?; @@ -406,7 +416,7 @@ pub(super) fn spawn_vshard_schedulers( // a no-op once the set has converged. let shared_task = Arc::clone(shared); let sm_task = Arc::clone(sequencer_state_machine); - let rr_task = Arc::clone(calvin_read_result_senders); + let served_task = Arc::clone(&served_vshards); let registry_task = Arc::clone(calvin_completion_registry); let cfg_task = scheduler_config.clone(); crate::control::shutdown::spawn_loop( @@ -429,7 +439,7 @@ pub(super) fn spawn_vshard_schedulers( raft_loop_handle: &raft_loop_handle, sequencer_proposer: &sequencer_proposer, sequencer_state_machine: &sm_task, - calvin_read_result_senders: &rr_task, + served_vshards: &served_task, calvin_completion_registry: ®istry_task, scheduler_config: &cfg_task, }) { diff --git a/nodedb/src/control/distributed_applier/apply_loop/barrier_prefold.rs b/nodedb/src/control/distributed_applier/apply_loop/barrier_prefold.rs new file mode 100644 index 000000000..06d8ebe57 --- /dev/null +++ b/nodedb/src/control/distributed_applier/apply_loop/barrier_prefold.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Fold the dependent-read barrier entries of a waiting lane ahead of their +//! turn. +//! +//! A group's entries start in log order. A write held for this node's +//! metadata apply (see [`super::metadata_floor`]) keeps every later entry of +//! its group in the backlog until the catalog catches up. A barrier entry +//! names no collection and needs no catalog. Behind such a hold it would +//! still wait, and the txn's barrier on this replica would lack an event its +//! peers folded. +//! +//! So each pump folds every barrier entry the backlog holds past the last +//! one it folded, in log order. Barrier entries fold in log order among +//! themselves, which is the only order a barrier log depends on. Each entry +//! still concludes at its turn (see +//! [`super::calvin_read_result::prefold_barrier_entry`]). + +use super::calvin_read_result::prefold_barrier_entry; +use super::context::ApplyContext; +use super::lane::Lane; + +/// Fold each barrier entry of `lane`'s backlog above +/// `lane.prefolded_through`, in log order, and move the mark past every +/// backlog entry. +pub(super) fn prefold_backlog(ctx: ApplyContext<'_>, lane: &mut Lane, group_id: u64) { + let through = lane.prefolded_through; + let mut last = through; + for queued in lane.backlog.iter() { + let index = queued.entry.index; + if index <= through { + continue; + } + last = last.max(index); + if let Some(decoded) = queued.decoded.as_ref() { + prefold_barrier_entry(ctx, group_id, index, decoded); + } + } + lane.prefolded_through = last; +} diff --git a/nodedb/src/control/distributed_applier/apply_loop/calvin_read_result.rs b/nodedb/src/control/distributed_applier/apply_loop/calvin_read_result.rs index 998e8aa74..2ad1f2fca 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/calvin_read_result.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/calvin_read_result.rs @@ -1,25 +1,30 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Forward a committed `ReplicatedWrite::CalvinReadResult` entry to the local -//! Calvin scheduler's read-result channel for its target vShard. +//! Fold a committed `CalvinReadResult` or `CalvinReadTimeout` entry into the +//! barrier log of its txn on its target vShard. //! -//! A read result is forwarded to an in-memory Calvin scheduler and writes -//! nothing durable, so the caller neither advances the applied prefix nor -//! breaks it for this entry. Advancing on it would assert a redo record that -//! does not exist; breaking on it would stall the floor behind an entry that a -//! re-delivery could not usefully replay anyway — the epoch it belongs to does -//! not survive a restart — and force every later write in the batch to be -//! applied twice on the next boot. - -use std::collections::BTreeMap; -use std::sync::{Arc, Mutex}; - -use tokio::sync::mpsc; +//! The event lands in the txn's stored row first (see +//! [`crate::control::cluster::calvin::scheduler::barrier_store`]), then in +//! the vShard's read-result buffer, which wakes the scheduler. An event that +//! applies before this node granted the txn, or before a scheduler for the +//! vShard runs here, waits in the buffer or its row: none is dropped. An +//! event for a position the vShard's applied ledger holds is dropped: the +//! txn finished, and no barrier of it opens again. +//! +//! The entry counts as durably applied once its row holds the event. Boot +//! resumes Raft delivery above the group's durable applied floor, so the +//! row is what a barrier reads after a restart. A row write that fails +//! leaves the event in memory alone, and the entry breaks the applied +//! prefix: a restart delivers it again. use crate::control::array_sync::raft_apply::AppliedPosition; -use crate::control::cluster::calvin::ReadResultEvent; -use crate::control::distributed_applier::propose_tracker::{AppliedWrite, ProposeTracker}; -use crate::types::TenantId; +use crate::control::cluster::calvin::BarrierEvent; +use crate::control::cluster::calvin::scheduler::barrier_store; +use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; +use crate::control::distributed_applier::propose_tracker::AppliedWrite; + +use super::context::ApplyContext; +use super::proposal_gate::EntryOutcome; /// Fields extracted from a `ReplicatedWrite::CalvinReadResult` entry. pub(super) struct CalvinReadResultFields<'a> { @@ -27,80 +32,452 @@ pub(super) struct CalvinReadResultFields<'a> { pub epoch: u64, pub position: u32, pub passive_vshard: u32, - pub tenant_id: u64, pub values: &'a [u8], } -/// Decode `fields.values`, forward the resulting [`ReadResultEvent`] to the -/// scheduler registered for `fields.target_vshard`, and complete the propose -/// waiter. +/// Decode `fields.values` and fold the read result into the barrier log of +/// its txn on `fields.target_vshard`, then complete the propose waiter. pub(super) fn forward_calvin_read_result( - tracker: &Arc, - calvin_read_result_senders: &Arc>>>, + ctx: ApplyContext<'_>, pos: AppliedPosition, fields: CalvinReadResultFields<'_>, -) { - let AppliedPosition { - group_id, - log_index, - applied_key, - .. - } = pos; - - let decoded_values: Vec<( +) -> EntryOutcome { + let values: Vec<( nodedb_physical::physical_plan::meta::PassiveReadKeyId, nodedb_types::Value, )> = match zerompk::from_msgpack(fields.values) { Ok(decoded) => decoded, Err(e) => { + // Every replica decodes the same bytes and refuses them alike, + // so each barrier of the txn lacks this result and waits for + // the log's timeout entry. A restart refuses them again, so the + // refusal holds nothing to keep. tracing::warn!( - group_id, - index = log_index, + group_id = pos.group_id, + index = pos.log_index, error = %e, "failed to decode CalvinReadResult payload" ); - tracker.complete( - group_id, - log_index, - applied_key, - Err(crate::Error::Internal { - detail: format!("decode CalvinReadResult payload: {e}"), + ctx.tracker.complete( + pos.group_id, + pos.log_index, + pos.applied_key, + Err(crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("CalvinReadResult payload: {e}"), }), ); - // Prefix-neutral, like the forward below: a read result mints no - // durable state either way, so there is nothing a re-delivery - // could restore and nothing later entries must wait behind. - return; + return EntryOutcome::Applied { + durable: true, + result: None, + }; } }; + fold_barrier_event( + ctx, + pos, + fields.target_vshard, + TxnId::new(fields.epoch, fields.position), + BarrierEvent::Read { + passive_vshard: fields.passive_vshard, + values, + }, + ) +} + +/// Fold a `CalvinReadTimeout` entry for txn `(epoch, position)` into its +/// barrier log on `target_vshard`, then complete the propose waiter. +pub(super) fn forward_calvin_read_timeout( + ctx: ApplyContext<'_>, + pos: AppliedPosition, + target_vshard: u32, + epoch: u64, + position: u32, +) -> EntryOutcome { + fold_barrier_event( + ctx, + pos, + target_vshard, + TxnId::new(epoch, position), + BarrierEvent::Timeout, + ) +} + +/// Fold `event` for `txn` into its stored row and `target_vshard`'s buffer, +/// unless the txn finished on this replica, and complete the entry's +/// propose waiter. +fn fold_barrier_event( + ctx: ApplyContext<'_>, + pos: AppliedPosition, + target_vshard: u32, + txn: TxnId, + event: BarrierEvent, +) -> EntryOutcome { + if matches!(event, BarrierEvent::Read { .. }) { + ctx.state + .calvin + .read_results + .vshard(target_vshard) + .note_read_applied(pos.group_id, pos.log_index); + } + let saved = fold_unless_finished(ctx, target_vshard, txn, event); + tracing::debug!( + node_id = ctx.state.node_id, + group_id = pos.group_id, + index = pos.log_index, + vshard_id = target_vshard, + epoch = txn.epoch, + position = txn.position, + saved, + "calvin: barrier entry applied" + ); + // The event reached this replica's barrier either way: the waiter + // learns the entry committed. + ctx.tracker.complete( + pos.group_id, + pos.log_index, + pos.applied_key, + Ok(AppliedWrite::unversioned(Vec::new())), + ); + EntryOutcome::Applied { + durable: saved, + result: None, + } +} + +/// Fold `event` of `txn` into its stored row and `vshard_id`'s buffer, +/// unless the txn finished on this replica. Returns whether the row holds +/// the event, or needs none. +fn fold_unless_finished( + ctx: ApplyContext<'_>, + vshard_id: u32, + txn: TxnId, + event: BarrierEvent, +) -> bool { + if is_finished(ctx, vshard_id, txn) { + return true; + } + let saved = save_and_buffer(ctx, vshard_id, txn, event); + // The scheduler marks the ledger before it drops a finished txn's + // entries. A finish between the check above and the save saw no entry + // to drop, so this drops the one just saved. + if is_finished(ctx, vshard_id, txn) { + drop_finished(ctx, vshard_id, txn); + } + saved +} - let event = ReadResultEvent { - epoch: fields.epoch, - position: fields.position, - passive_vshard: fields.passive_vshard, - tenant_id: TenantId::new(fields.tenant_id), - values: decoded_values, +/// Fold the barrier event of `entry`, at `log_index` of `group_id`, ahead +/// of its turn in the group's lane. Entries that are not barrier entries +/// and payloads that do not decode change nothing. +/// +/// An earlier entry of the group can wait for this node's metadata apply, +/// and every later entry waits behind it. A barrier event names no +/// collection and needs no catalog, so it folds while the lane waits. Its +/// entry still concludes in log order, and that conclusion folds it again: +/// a second fold of an event the log already holds changes nothing. The +/// waiter, the applied counter and the durable prefix move only at the +/// conclusion. +/// +/// A barrier decision never runs ahead of the lane: a leader stages only +/// once its group applied its term's first entry. +pub(super) fn prefold_barrier_entry( + ctx: ApplyContext<'_>, + group_id: u64, + log_index: u64, + entry: &crate::control::wal_replication::ReplicatedEntry, +) { + use crate::control::wal_replication::ReplicatedWrite; + let (txn, event) = match &entry.write { + ReplicatedWrite::CalvinReadResult { + epoch, + position, + passive_vshard, + values, + .. + } => { + let Ok(values) = zerompk::from_msgpack(values) else { + // The conclusion reports the payload. + return; + }; + ( + TxnId::new(*epoch, *position), + BarrierEvent::Read { + passive_vshard: *passive_vshard, + values, + }, + ) + } + ReplicatedWrite::CalvinReadTimeout { + epoch, position, .. + } => (TxnId::new(*epoch, *position), BarrierEvent::Timeout), + _ => return, }; + let saved = fold_unless_finished(ctx, entry.vshard_id, txn, event); + tracing::debug!( + node_id = ctx.state.node_id, + group_id, + index = log_index, + vshard_id = entry.vshard_id, + epoch = txn.epoch, + position = txn.position, + saved, + "calvin: barrier entry folded ahead of its lane" + ); +} - let send_result = calvin_read_result_senders - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get(&fields.target_vshard) - .cloned() - .map(|sender| sender.try_send(event)); +/// Whether `txn` finished on `vshard_id` here: its position is applied. +fn is_finished(ctx: ApplyContext<'_>, vshard_id: u32, txn: TxnId) -> bool { + ctx.state + .calvin + .applied + .get(vshard_id) + .is_some_and(|ledger| ledger.is_applied(txn.epoch, txn.position)) +} - if let Some(Err(e)) = send_result { +/// Drop the buffered entries and the row of `txn` on `vshard_id`, which +/// finished. A failed remove leaves a row no barrier reads, and the +/// scheduler's stall-tick sweep removes it. +fn drop_finished(ctx: ApplyContext<'_>, vshard_id: u32, txn: TxnId) { + ctx.state + .calvin + .read_results + .vshard(vshard_id) + .forget_txn(txn); + if let Err(e) = barrier_store::remove_log(ctx.state.credentials.catalog(), vshard_id, txn) { tracing::warn!( - group_id, - index = log_index, + vshard_id, + epoch = txn.epoch, + position = txn.position, error = %e, - "failed to forward CalvinReadResult to scheduler" + "calvin: the barrier row of a finished txn was not removed" + ); + crate::diag::calvin_barrier_log_store_failed( + vshard_id, + Some((txn.epoch, txn.position)), + "remove", + &e, ); } - tracker.complete( - group_id, - log_index, - applied_key, - Ok(AppliedWrite::unversioned(Vec::new())), +} + +/// Save `event` in the row of `txn` on `vshard_id`, then hand it to the +/// vShard's buffer. Returns whether the row holds it. +/// +/// A txn with an event its row lacks skips the row: the row stays a prefix +/// of the txn's events, so a restart that delivers the missing entry again +/// folds them in log order. +fn save_and_buffer(ctx: ApplyContext<'_>, vshard_id: u32, txn: TxnId, event: BarrierEvent) -> bool { + let buffer = ctx.state.calvin.read_results.vshard(vshard_id); + let saved = !buffer.is_unsaved(txn) + && match barrier_store::save_event(ctx.state.credentials.catalog(), vshard_id, txn, &event) + { + Ok(()) => true, + Err(e) => { + tracing::error!( + vshard_id, + epoch = txn.epoch, + position = txn.position, + error = %e, + "calvin: the barrier event's row write failed; the event stays in memory" + ); + crate::diag::calvin_barrier_log_store_failed( + vshard_id, + Some((txn.epoch, txn.position)), + "save", + &e, + ); + false + } + }; + buffer.push( + txn, + event, + ctx.state.tuning.calvin.max_inflight_backlog, + saved, ); + saved +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use nodedb_physical::physical_plan::meta::PassiveReadKeyId; + use nodedb_types::{QualifiedCollection, Value}; + + use super::*; + use crate::bridge::dispatch::Dispatcher; + use crate::control::distributed_applier::propose_tracker::ProposeTracker; + use crate::control::state::SharedState; + use crate::wal::WalManager; + + const VSHARD: u32 = 6; + + fn pos(log_index: u64) -> AppliedPosition { + AppliedPosition { + group_id: 2, + log_index, + applied_key: 0, + commit_hlc: 0, + } + } + + fn values() -> Vec { + let values = vec![( + PassiveReadKeyId::kv( + QualifiedCollection::from_stored("items".to_owned()), + b"k".to_vec(), + ), + Value::Bytes(b"v".to_vec()), + )]; + zerompk::to_msgpack_vec(&values).expect("encode values") + } + + fn state() -> (Arc, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("wal")).expect("wal")); + let (dispatcher, _sides) = Dispatcher::new(1, 64); + ( + SharedState::new(dispatcher, wal).expect("shared state"), + dir, + ) + } + + /// A read result for a txn no scheduler here holds waits in the buffer. + #[test] + fn a_result_before_the_grant_waits_in_the_buffer() { + let (state, _dir) = state(); + let tracker = Arc::new(ProposeTracker::new()); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + }; + let bytes = values(); + let outcome = forward_calvin_read_result( + ctx, + pos(10), + CalvinReadResultFields { + target_vshard: VSHARD, + epoch: 4, + position: 0, + passive_vshard: 9, + values: &bytes, + }, + ); + assert!(matches!( + outcome, + EntryOutcome::Applied { + durable: true, + result: None + } + )); + forward_calvin_read_timeout(ctx, pos(11), VSHARD, 4, 0); + assert_eq!( + state.calvin.read_results.vshard(VSHARD).read_entries(), + vec![(2, 10)], + "the buffer names the read result's entry, not the timeout's" + ); + assert_eq!(state.calvin.read_results.waiting_txns(VSHARD), 1); + let events = state + .calvin + .read_results + .vshard(VSHARD) + .take(TxnId::new(4, 0)) + .expect("the buffered log"); + assert!(!events.stored); + assert!(events.memory.missing(&[9].into_iter().collect()).is_empty()); + // The row holds both events: a restart reads them from it. + let stored = barrier_store::load_log(state.credentials.catalog(), VSHARD, TxnId::new(4, 0)) + .expect("load the row"); + assert_eq!(stored, events.memory); + } + + /// A barrier entry folds ahead of its turn while its lane waits, and + /// its conclusion folds it again with no change. Only the conclusion + /// counts the entry and completes its waiter. + #[test] + fn a_prefolded_entry_concludes_without_a_second_change() { + let (state, _dir) = state(); + let tracker = Arc::new(ProposeTracker::new()); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + }; + let bytes = values(); + let entry = crate::control::wal_replication::ReplicatedEntry::new( + 1, + 0, + VSHARD, + crate::control::wal_replication::ReplicatedWrite::CalvinReadResult { + epoch: 5, + position: 0, + passive_vshard: 9, + tenant_id: 1, + values: bytes.clone(), + }, + ); + prefold_barrier_entry(ctx, 2, 12, &entry); + let buffer = state.calvin.read_results.vshard(VSHARD); + assert_eq!(buffer.waiting_txns(), 1, "the event waits before its turn"); + assert!(buffer.read_entries().is_empty(), "a prefold counts nothing"); + let stored = barrier_store::load_log(state.credentials.catalog(), VSHARD, TxnId::new(5, 0)) + .expect("load the row"); + assert!(stored.missing(&[9].into_iter().collect()).is_empty()); + + let outcome = forward_calvin_read_result( + ctx, + pos(12), + CalvinReadResultFields { + target_vshard: VSHARD, + epoch: 5, + position: 0, + passive_vshard: 9, + values: &bytes, + }, + ); + assert!(matches!( + outcome, + EntryOutcome::Applied { + durable: true, + result: None + } + )); + assert_eq!(buffer.read_entries(), vec![(2, 12)]); + assert_eq!( + barrier_store::load_log(state.credentials.catalog(), VSHARD, TxnId::new(5, 0)) + .expect("load the row"), + stored + ); + let events = buffer.take(TxnId::new(5, 0)).expect("buffered events"); + assert_eq!(events.memory, stored); + } + + /// An event for a position the vShard's ledger holds is dropped: the + /// txn finished. + #[test] + fn an_event_of_a_finished_txn_is_dropped() { + let (state, _dir) = state(); + state + .calvin + .applied + .get_or_create(VSHARD) + .mark_terminal(4, 0); + let tracker = Arc::new(ProposeTracker::new()); + let ctx = ApplyContext { + state: &state, + tracker: &tracker, + }; + forward_calvin_read_timeout(ctx, pos(11), VSHARD, 4, 0); + assert_eq!(state.calvin.read_results.waiting_txns(VSHARD), 0); + assert!( + state + .credentials + .catalog() + .load_calvin_barrier_log(VSHARD, 4, 0) + .expect("load") + .is_none(), + "a finished txn's event writes no row" + ); + } } diff --git a/nodedb/src/control/distributed_applier/apply_loop/context.rs b/nodedb/src/control/distributed_applier/apply_loop/context.rs index 21b27a800..3c7996901 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/context.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/context.rs @@ -3,28 +3,20 @@ //! What every entry's apply borrows from the loop, and the futures the loop //! collects while it prepares later entries. -use std::collections::BTreeMap; use std::future::Future; use std::pin::Pin; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; -use tokio::sync::mpsc; - -use crate::control::cluster::calvin::ReadResultEvent; use crate::control::distributed_applier::propose_tracker::ProposeTracker; use crate::control::state::SharedState; use super::proposal_gate::EntryOutcome; -/// Senders to each local Calvin scheduler's read-result channel, by vShard. -pub(super) type CalvinReadResultSenders = Arc>>>; - /// The loop-owned handles an entry's apply borrows. #[derive(Clone, Copy)] pub(super) struct ApplyContext<'a> { pub state: &'a Arc, pub tracker: &'a Arc, - pub calvin_read_result_senders: &'a CalvinReadResultSenders, } /// An entry whose apply finished: its waiter is resolved, and its outcome diff --git a/nodedb/src/control/distributed_applier/apply_loop/driver.rs b/nodedb/src/control/distributed_applier/apply_loop/driver.rs index 865cf9ffe..abdba3b87 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/driver.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/driver.rs @@ -8,7 +8,6 @@ use std::sync::Arc; use tokio::sync::{mpsc, watch}; -use crate::control::cluster::calvin::ReadResultEvent; use crate::control::distributed_applier::applier::ApplyBatch; use crate::control::distributed_applier::proposal_ledger::{ PROPOSAL_LEDGER_CAPACITY, ProposalLedger, @@ -27,9 +26,6 @@ pub async fn run_apply_loop( mut apply_rx: mpsc::Receiver, state: Arc, tracker: Arc, - calvin_read_result_senders: Arc< - std::sync::Mutex>>, - >, ) { // Proposals this node already applied, recovered from its WAL before any // entry is delivered: every record an entry's apply appended carries the @@ -57,7 +53,6 @@ pub async fn run_apply_loop( let ctx = ApplyContext { state: &state, tracker: &tracker, - calvin_read_result_senders: &calvin_read_result_senders, }; let mut pipeline = Pipeline::new(ctx, ledger); let mut install_released = state.raft_apply_gates.get().map(|g| g.subscribe_released()); diff --git a/nodedb/src/control/distributed_applier/apply_loop/lane.rs b/nodedb/src/control/distributed_applier/apply_loop/lane.rs index 956f0c28d..9cf667612 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/lane.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/lane.rs @@ -53,8 +53,8 @@ impl QueuedEntry { } /// `(tenant_id, write_hlc, restore_id)` of an entry that writes a - /// tenant's data. A cut barrier, a Calvin read result and a surrogate - /// bind write no data, a Calvin slice that raises no write mark writes + /// tenant's data. A cut barrier, a Calvin read result or read timeout + /// and a surrogate bind write no data, a Calvin slice that raises no write mark writes /// only derived rows or schema, and an entry with no proposer stamp has /// no commit HLC to record. pub fn write_stamp(&self) -> Option<(u64, u64, u64)> { @@ -64,6 +64,7 @@ impl QueuedEntry { decoded.write, ReplicatedWrite::CutBarrier { .. } | ReplicatedWrite::CalvinReadResult { .. } + | ReplicatedWrite::CalvinReadTimeout { .. } | ReplicatedWrite::SurrogateBind { .. } | ReplicatedWrite::RedoChunk { .. } | ReplicatedWrite::RedoAbandon { .. } @@ -98,6 +99,7 @@ impl QueuedEntry { ReplicatedWrite::ArraySchema { .. } | ReplicatedWrite::CutBarrier { .. } | ReplicatedWrite::CalvinReadResult { .. } + | ReplicatedWrite::CalvinReadTimeout { .. } | ReplicatedWrite::SurrogateBind { .. } | ReplicatedWrite::RedoChunk { .. } | ReplicatedWrite::RedoAbandon { .. } => false, @@ -178,6 +180,9 @@ pub(super) struct Lane { saved_floor: Option, /// The entry the last settle ended at. last_settled: Option, + /// The highest backlog index whose barrier event folded ahead of its + /// turn (see `barrier_prefold`). + pub prefolded_through: u64, } impl Lane { @@ -190,6 +195,7 @@ impl Lane { prefix: AppliedPrefix::new(), saved_floor: None, last_settled: None, + prefolded_through: 0, } } diff --git a/nodedb/src/control/distributed_applier/apply_loop/metadata_floor.rs b/nodedb/src/control/distributed_applier/apply_loop/metadata_floor.rs index d0b870dc8..9194ee802 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/metadata_floor.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/metadata_floor.rs @@ -78,6 +78,13 @@ pub(super) fn hold_for_metadata<'a>( /// order it holds. It fails only when the metadata group left this node. async fn await_metadata_floor(state: &SharedState, held: HeldEntry) -> crate::Result<()> { let watcher = state.applied_index_watcher(METADATA_GROUP_ID); + tracing::debug!( + group_id = held.group_id, + log_index = held.log_index, + metadata_floor = held.metadata_floor, + metadata_applied = watcher.current(), + "a replicated write holds its group's lane for this node's metadata apply" + ); loop { let waiting = Arc::clone(&watcher); let floor = held.metadata_floor; diff --git a/nodedb/src/control/distributed_applier/apply_loop/mod.rs b/nodedb/src/control/distributed_applier/apply_loop/mod.rs index 2423e526f..031510b6d 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/mod.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/mod.rs @@ -23,8 +23,11 @@ //! restore point's place, and a database backup's capture. //! - [`context`]: the handles an apply borrows, and the futures the loop //! collects. -//! - [`calvin_read_result`]: forwards a committed `CalvinReadResult` entry to -//! the local Calvin scheduler. +//! - [`barrier_prefold`]: folds the barrier entries of a waiting lane ahead +//! of their turn. +//! - [`calvin_read_result`]: folds a committed `CalvinReadResult` or +//! `CalvinReadTimeout` entry into its txn's stored barrier row and its +//! vShard's read-result buffer. //! - [`calvin_redo`]: claims a stamped Calvin redo's position in the applied //! ledger and reports its install to the vShard's scheduler. //! - [`write_dispatch`]: the generic decode + write-funnel enqueue path. @@ -49,6 +52,7 @@ //! collection it names holds the incarnation its proposer planned against. mod array_cell_route; +mod barrier_prefold; mod bookkeeping; mod calvin_read_result; mod calvin_redo; diff --git a/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs b/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs index 033bc6937..f99e02c5e 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/pipeline.rs @@ -183,6 +183,13 @@ impl<'a> Pipeline<'a> { .collect(); for group_id in groups { self.pump_group(group_id); + // The entries left wait behind an earlier one. Their barrier + // events fold now: a barrier needs no catalog. + if !self.install_blocked + && let Some(lane) = self.lanes.get_mut(&group_id) + { + super::barrier_prefold::prefold_backlog(self.ctx, lane, group_id); + } } } diff --git a/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs b/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs index e9064afd4..1c1a97ec1 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs @@ -279,8 +279,7 @@ pub(super) fn prepare_chunked_final<'a>( #[cfg(test)] mod tests { - use std::collections::BTreeMap; - use std::sync::{Arc, Mutex}; + use std::sync::Arc; use nodedb_physical::physical_plan::RedoOrigin; @@ -381,11 +380,9 @@ mod tests { let (dispatcher, _sides) = Dispatcher::new(1, 64); let state = SharedState::new(dispatcher, wal).expect("shared state"); let tracker = Arc::new(ProposeTracker::new()); - let senders = Arc::new(Mutex::new(BTreeMap::new())); let ctx = ApplyContext { state: &state, tracker: &tracker, - calvin_read_result_senders: &senders, }; let first = apply_chunk(ctx, pos(GROUP, 10), 0).await; @@ -416,11 +413,9 @@ mod tests { let (dispatcher, _sides) = Dispatcher::new(1, 64); let state = SharedState::new(dispatcher, wal).expect("shared state"); let tracker = Arc::new(ProposeTracker::new()); - let senders = Arc::new(Mutex::new(BTreeMap::new())); let ctx = ApplyContext { state: &state, tracker: &tracker, - calvin_read_result_senders: &senders, }; let Prepared::Concluded(outcome) = diff --git a/nodedb/src/control/distributed_applier/apply_loop/start.rs b/nodedb/src/control/distributed_applier/apply_loop/start.rs index f20dd6cfd..f7563d87c 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/start.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/start.rs @@ -18,7 +18,9 @@ use crate::control::backup::cut_order::OrderedCut; use crate::control::wal_replication::{ReplicatedEntry, ReplicatedWrite}; use crate::types::{DatabaseId, TenantId}; -use super::calvin_read_result::{CalvinReadResultFields, forward_calvin_read_result}; +use super::calvin_read_result::{ + CalvinReadResultFields, forward_calvin_read_result, forward_calvin_read_timeout, +}; use super::context::{ApplyContext, ApplyFuture, EnqueueFuture, FinishedApply}; use super::group_watch::GroupWatch; use super::lane::QueuedEntry; @@ -275,28 +277,34 @@ fn prepare_replicated<'a>( epoch, position, passive_vshard, - tenant_id, + tenant_id: _, ref values, } => { - forward_calvin_read_result( - ctx.tracker, - ctx.calvin_read_result_senders, + // The event's stored row is its durable effect: the entry + // extends the applied prefix once the row holds it. + Prepared::Concluded(forward_calvin_read_result( + ctx, pos, CalvinReadResultFields { target_vshard: replicated.vshard_id, epoch, position, passive_vshard, - tenant_id, values, }, - ); - // A read result is forwarded to an in-memory Calvin scheduler and - // writes nothing durable, so it neither advances the prefix nor - // breaks it. The epoch it belongs to does not survive a restart, - // so a re-delivery cannot usefully replay it. - Prepared::Concluded(EntryOutcome::Skipped) + )) } + ReplicatedWrite::CalvinReadTimeout { + epoch, + position, + tenant_id: _, + } => Prepared::Concluded(forward_calvin_read_timeout( + ctx, + pos, + replicated.vshard_id, + epoch, + position, + )), _ => prepare_generic_entry(ctx, pos, entry, scope, false), } } diff --git a/nodedb/src/control/gateway/cluster_error.rs b/nodedb/src/control/gateway/cluster_error.rs index 66c2c7210..32c462e10 100644 --- a/nodedb/src/control/gateway/cluster_error.rs +++ b/nodedb/src/control/gateway/cluster_error.rs @@ -42,9 +42,15 @@ pub(super) fn map_typed_cluster_error(err: TypedClusterError, vshard_id: u64) -> descriptor: collection, } } - TypedClusterError::DeadlineExceeded { .. } => Error::DeadlineExceeded { - request_id: crate::types::RequestId::new(0), - }, + TypedClusterError::DeadlineExceeded { elapsed_ms } => { + tracing::warn!( + elapsed_ms, + "gateway: a remote node reported the request's deadline exceeded" + ); + Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + } + } // Remote Data-Plane verdict: keep the code so the client sees the // SQLSTATE local execution renders, not a generic internal error. TypedClusterError::DataPlane { code } => Error::DataPlane(code.into()), diff --git a/nodedb/src/control/gateway/dispatch_remote.rs b/nodedb/src/control/gateway/dispatch_remote.rs index 6ea4cea28..ec627d6e2 100644 --- a/nodedb/src/control/gateway/dispatch_remote.rs +++ b/nodedb/src/control/gateway/dispatch_remote.rs @@ -174,10 +174,29 @@ pub(super) async fn dispatch_remote( "gateway: dispatching ExecuteRequest to remote node" ); - let resp_rpc = transport - .send_rpc(node_id, req) - .await - .map_err(|e| send_error(e, resend_safe, node_id, vshard_id))?; + // The remote handler works within `deadline_ms`: a write can wait at the + // leader's gate or in a Calvin queue for that long. The reply wait follows + // that budget, never the transport's fixed RPC timeout. + let reply_wait = nodedb_cluster::rpc_codec::reply_wait(deadline_ms); + let resp_rpc = match tokio::time::timeout( + reply_wait, + transport.send_rpc_with_read_timeout(node_id, req, reply_wait), + ) + .await + { + Ok(answer) => answer.map_err(|e| send_error(e, resend_safe, node_id, vshard_id))?, + Err(_) => { + return Err(send_error( + ClusterError::ShardTimeout { + vshard_id: (vshard_id % VShardId::COUNT as u64) as u32, + elapsed_ms: u64::try_from(reply_wait.as_millis()).unwrap_or(u64::MAX), + }, + resend_safe, + node_id, + vshard_id, + )); + } + }; match resp_rpc { RaftRpc::ExecuteResponse(resp) => { @@ -466,6 +485,12 @@ fn send_error(e: ClusterError, resend_safe: bool, node_id: u64, vshard_id: u64) ClusterError::Unanswered { .. } | ClusterError::ShardTimeout { .. } ); if outcome_unknown && !resend_safe { + tracing::warn!( + node_id, + vshard_id, + error = %e, + "gateway: a remote execute reached its deadline with no answer" + ); return Error::DeadlineExceeded { request_id: crate::types::RequestId::new(0), }; diff --git a/nodedb/src/control/planner/calvin/dependent_recon.rs b/nodedb/src/control/planner/calvin/dependent_recon.rs index ca77a2230..8bbbb277b 100644 --- a/nodedb/src/control/planner/calvin/dependent_recon.rs +++ b/nodedb/src/control/planner/calvin/dependent_recon.rs @@ -23,7 +23,7 @@ use crate::Error; use crate::control::cluster::calvin::executor::ollp::error::OllpError; use crate::control::planner::calvin::preexec::PreexecScan; use crate::control::planner::calvin::{ - DependentOutcome, DependentRetryArgs, build_single_vshard_dependent_tx_class, + DependentOutcome, DependentRetryArgs, build_single_vshard_predicted_tx_class, is_dependent_predicate, predicate_class_for_filters, run_dependent_with_retry, submit_calvin_routed_assign, }; @@ -146,7 +146,7 @@ pub fn is_edge_recon_plan(plan: &nodedb_physical::physical_plan::PhysicalPlan) - /// [`plan_needs_implicit_edge_recon`]) so it does not have to be re-derived. /// /// The `TxClass` this builds accepts a write set on one vShard -/// ([`build_single_vshard_dependent_tx_class`]). A delete whose row, node +/// ([`build_single_vshard_predicted_tx_class`]). A delete whose row, node /// guard and edges all home on one vShard is a legitimate one-vShard /// transaction, and so is a contended single-collection predicate write /// routed here by the write-admission gate. @@ -384,7 +384,7 @@ async fn dispatch_dependent_edge_recon_inner( t }) .collect(); - let tx_class = build_single_vshard_dependent_tx_class( + let tx_class = build_single_vshard_predicted_tx_class( &modified_tasks, tenant_id, dep_collection, diff --git a/nodedb/src/control/planner/calvin/mod.rs b/nodedb/src/control/planner/calvin/mod.rs index a37011a80..b0feb9ac1 100644 --- a/nodedb/src/control/planner/calvin/mod.rs +++ b/nodedb/src/control/planner/calvin/mod.rs @@ -16,6 +16,7 @@ pub mod explain; pub mod node_delete_txn; pub mod predicate; pub mod preexec; +pub mod read_dependent; pub mod reservation; pub mod retry_loop; pub mod submit; @@ -46,7 +47,7 @@ pub use submit::{ submit_calvin_routed, submit_calvin_routed_assign, submit_calvin_routed_write, }; pub use tx_class::{ - build_dependent_tx_class, build_single_vshard_dependent_tx_class, build_single_vshard_tx_class, - build_static_tx_class, + PassiveReads, build_predicted_tx_class, build_read_dependent_tx_class, + build_single_vshard_predicted_tx_class, build_single_vshard_tx_class, build_static_tx_class, }; pub use types::{DispatchClass, DispatchOutcome, TxnDispatchPosition}; diff --git a/nodedb/src/control/planner/calvin/read_dependent/item_move.rs b/nodedb/src/control/planner/calvin/read_dependent/item_move.rs new file mode 100644 index 000000000..05ef4a5e0 --- /dev/null +++ b/nodedb/src/control/planner/calvin/read_dependent/item_move.rs @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `TRANSFER_ITEM` across two vShards, as one read-dependent Calvin +//! transaction. +//! +//! The item leaves the source collection's vShard and arrives at the +//! destination collection's vShard with the bytes it held at the source. +//! The destination write depends on a row of another vShard, so: +//! +//! 1. The coordinator reads the item at its source owner. +//! 2. It plans a delete of the item at the source and a put of the same +//! bytes at the destination, and submits them as one transaction whose +//! passive read is the item, with the bytes it read as the expected value. +//! 3. Under the transaction's locks the source vShard reads the item again +//! and broadcasts it to both active vShards through their data-group logs. +//! Each votes `PredictionDrift` when the item moved, and the coordinator +//! reads again and resubmits. An item gone at the source ends the move +//! with `NotFound`. +//! +//! Both halves commit, or neither does. + +use std::collections::BTreeMap; + +use nodedb_cluster::calvin::types::{EngineKeySet, PassiveReadKeyId, SortedVec}; +use nodedb_physical::physical_plan::{KvOp, PhysicalPlan}; +use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; +use nodedb_types::{CollectionKey, QualifiedCollection, RlsWriteCheck, Surrogate}; + +use super::recon::read_kv_row; +use crate::Error; +use crate::bridge::envelope::ErrorCode; +use crate::control::cluster::calvin::executor::ollp::error::OllpError; +use crate::control::planner::calvin::dependent_recon_finish::finish_committed; +use crate::control::planner::calvin::{ + DependentOutcome, DependentRetryArgs, PassiveReads, build_read_dependent_tx_class, + predicate_class, run_dependent_with_retry, submit_calvin_routed_assign, +}; +use crate::control::planner::rls_injection::inject_rls_for_single_plan; +use crate::control::security::audit::ArcAuditEmitter; +use crate::control::security::identity::AuthenticatedIdentity; +use crate::control::security::request_scope::RequestAuthScope; +use crate::control::server::shared::authorization::authorize_task_set; +use crate::control::state::SharedState; +use crate::types::{DatabaseId, TenantId}; + +/// One item move between collections on two vShards. +pub struct CrossShardItemMove { + pub tenant_id: TenantId, + pub database_id: DatabaseId, + pub source_collection: QualifiedCollection, + pub dest_collection: QualifiedCollection, + /// The item's key at the source. + pub item_key: Vec, + /// The item's key at the destination. + pub dest_key: Vec, + /// The moved row's identity at the destination. + pub surrogate: Surrogate, + /// The source collection's compiled write policy, decided against the + /// row the move removes. + pub source_rls_write_check: RlsWriteCheck, +} + +/// Move the item of `mv` across its two vShards as one read-dependent +/// Calvin transaction, for `identity`. +/// +/// Returns once both halves committed. An item absent at the source ends +/// the move with `ErrorCode::NotFound`, as a same-vShard move does. A write +/// policy of the destination that refuses the item's bytes refuses the move +/// before anything is submitted. +pub async fn move_item_across_shards( + state: &SharedState, + identity: &AuthenticatedIdentity, + mv: CrossShardItemMove, +) -> crate::Result<()> { + let registry = state + .calvin_completion_registry + .get() + .ok_or(Error::SequencerUnavailable)?; + let orchestrator = state + .ollp_orchestrator + .get() + .ok_or(Error::SequencerUnavailable)?; + let source_vshard = + CollectionKey::from_qualified_str(mv.database_id, mv.source_collection.as_str())?.vshard(); + let dest_vshard = + CollectionKey::from_qualified_str(mv.database_id, mv.dest_collection.as_str())?.vshard(); + + let delete_task = PhysicalTask { + tenant_id: mv.tenant_id, + vshard_id: source_vshard, + database_id: mv.database_id, + plan: PhysicalPlan::Kv(KvOp::Delete { + collection: mv.source_collection.clone(), + keys: vec![mv.item_key.clone()], + rls_write_check: mv.source_rls_write_check.clone(), + returning: None, + rls_filters: Vec::new(), + provenance: None, + }), + post_set_op: PostSetOp::None, + txn_id: None, + }; + let put_task = |value: Vec| PhysicalTask { + tenant_id: mv.tenant_id, + vshard_id: dest_vshard, + database_id: mv.database_id, + plan: PhysicalPlan::Kv(KvOp::Put { + collection: mv.dest_collection.clone(), + key: mv.dest_key.clone(), + value, + ttl_ms: 0, + surrogate: mv.surrogate, + returning: None, + rls_filters: Vec::new(), + provenance: None, + }), + post_set_op: PostSetOp::None, + txn_id: None, + }; + let passive_keys = BTreeMap::from([( + source_vshard.as_u32(), + vec![EngineKeySet::Kv { + collection: mv.source_collection.as_str().to_owned(), + keys: SortedVec::new(vec![mv.item_key.clone()]), + }], + )]); + let item_row = PassiveReadKeyId::kv(mv.source_collection.clone(), mv.item_key.clone()); + let class_hash = predicate_class( + &format!("TRANSFER_ITEM TO {}", mv.dest_collection.as_str()), + mv.source_collection.as_str(), + ); + let tenant_id = mv.tenant_id; + let source = &mv; + // Copy: it captures only references. + let read_item = move || async move { + read_kv_row( + state, + source.tenant_id, + source.database_id, + &source.source_collection, + &source.item_key, + ) + .await? + .ok_or(Error::DataPlane(ErrorCode::NotFound)) + }; + + let submit = |item: &Vec| { + let item = item.clone(); + let delete_task = delete_task.clone(); + let passive_keys = passive_keys.clone(); + let item_row = item_row.clone(); + let put = put_task(item.clone()); + async move { + let put = admit_destination_image(state, identity, put) + .map_err(|e| OllpError::Terminal(Box::new(e)))?; + // Each attempt's tasks pass the authorization boundary as the + // capability they are submitted under. + let emitter = ArcAuditEmitter(std::sync::Arc::clone(&state.audit)); + let tasks: Vec = authorize_task_set( + identity, + &[delete_task, put], + &state.permissions, + &state.roles, + &emitter, + ) + .map_err(|e| OllpError::Terminal(Box::new(e.into())))? + .into_tasks() + .into_iter() + .map(|task| task.into_physical_task()) + .collect(); + let reads = PassiveReads { + keys: passive_keys, + expected: BTreeMap::from([(item_row, Some(item))]), + }; + orchestrator + .submit_with_retry_via( + class_hash, + tenant_id, + || { + build_read_dependent_tx_class(&tasks, tenant_id, reads.clone()) + .map(Some) + .map_err(|e| OllpError::Terminal(Box::new(e))) + }, + |tx_class| async move { + submit_calvin_routed_assign(state, tx_class) + .await + .map_err(|e| OllpError::Retryable(Box::new(e))) + }, + ) + .await + } + }; + + let initial_item = read_item().await?; + let outcome = run_dependent_with_retry(DependentRetryArgs { + registry, + orchestrator, + predicate_class_hash: class_hash, + timeout: std::time::Duration::from_secs(state.tuning.network.default_deadline_secs), + ollp_max_retries: u32::from(orchestrator.ollp_max_retries()), + initial_predicted: initial_item, + submit, + rescan: read_item, + }) + .await?; + let (txn_id, ack_results) = match outcome { + DependentOutcome::Committed { + txn_id, + ack_results, + } => (txn_id, ack_results), + DependentOutcome::NoOp => { + return Err(Error::Internal { + detail: "a cross-shard TRANSFER_ITEM submitted no transaction".to_owned(), + }); + } + }; + // The applied responses of the two writes are counts the move does not + // report. Draining them keeps the sidecar from holding them. The drain + // reads only the plans' collections and kinds. + finish_committed( + state, + &[delete_task, put_task(Vec::new())], + txn_id, + &ack_results, + ) + .await?; + Ok(()) +} + +/// Run the destination's write policy over the item's bytes in `put`: the +/// injection pass admits the image, or refuses the move. +fn admit_destination_image( + state: &SharedState, + identity: &AuthenticatedIdentity, + mut put: PhysicalTask, +) -> crate::Result { + let scope = RequestAuthScope::for_database(identity, state.auth_stores(), put.database_id); + inject_rls_for_single_plan( + put.tenant_id.as_u64(), + put.database_id, + &mut put.plan, + &state.rls, + state.credentials.catalog(), + scope.auth(), + )?; + Ok(put) +} diff --git a/nodedb/src/control/planner/calvin/read_dependent/mod.rs b/nodedb/src/control/planner/calvin/read_dependent/mod.rs new file mode 100644 index 000000000..afbea4442 --- /dev/null +++ b/nodedb/src/control/planner/calvin/read_dependent/mod.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Read-dependent Calvin transactions: cross-shard writes whose values +//! depend on rows of another vShard. + +pub mod item_move; +mod recon; + +pub use item_move::{CrossShardItemMove, move_item_across_shards}; diff --git a/nodedb/src/control/planner/calvin/read_dependent/recon.rs b/nodedb/src/control/planner/calvin/read_dependent/recon.rs new file mode 100644 index 000000000..837af4310 --- /dev/null +++ b/nodedb/src/control/planner/calvin/read_dependent/recon.rs @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The coordinator's read of a row a read-dependent transaction depends on. +//! +//! The read reaches the row's vShard owner through the gateway, as a +//! `SELECT` does, and returns the stored bytes a passive read of the same +//! row returns under the transaction's locks. The active participants +//! compare the two, so the read carries no row filter: both sides see the +//! row as stored. + +use nodedb_physical::physical_plan::{KvOp, PhysicalPlan}; +use nodedb_types::QualifiedCollection; + +use crate::control::state::SharedState; +use crate::types::{DatabaseId, TenantId, TraceId}; + +/// The stored bytes of key-value row `key` of `collection`, read on the +/// collection's vShard owner. `None` when the row is absent. +pub(super) async fn read_kv_row( + state: &SharedState, + tenant_id: TenantId, + database_id: DatabaseId, + collection: &QualifiedCollection, + key: &[u8], +) -> crate::Result>> { + let plan = PhysicalPlan::Kv(KvOp::Get { + collection: collection.clone(), + key: key.to_vec(), + rls_filters: Vec::new(), + surrogate_ceiling: None, + }); + let gateway = state.installed_gateway()?; + let ctx = crate::control::gateway::core::QueryContext { + tenant_id, + trace_id: TraceId::ZERO, + database_id, + txn_id: None, + linearizable: true, + }; + let payloads = gateway.execute_internal(&ctx, plan).await?; + // A KV get routes to the collection's one vShard. An absent row answers + // with an empty payload: a stored row is a msgpack image, never empty. + Ok(payloads + .into_iter() + .next() + .filter(|bytes| !bytes.is_empty())) +} diff --git a/nodedb/src/control/planner/calvin/submit/assign.rs b/nodedb/src/control/planner/calvin/submit/assign.rs index a4a5bca27..f2781ac71 100644 --- a/nodedb/src/control/planner/calvin/submit/assign.rs +++ b/nodedb/src/control/planner/calvin/submit/assign.rs @@ -211,7 +211,13 @@ pub async fn submit_calvin_routed_assign( } // The submit went out and its answer was lost. The leader can have // assigned it, so its outcome is unknown. - Err(ClusterError::Unanswered { .. }) => { + Err(error @ ClusterError::Unanswered { .. }) => { + tracing::warn!( + leader, + %error, + "calvin-inbox submit reached its deadline with no answer from the sequencer \ + leader" + ); return Err(Error::DeadlineExceeded { request_id: crate::types::RequestId::new(0), }); diff --git a/nodedb/src/control/planner/calvin/submit/routed.rs b/nodedb/src/control/planner/calvin/submit/routed.rs index 45bfdac59..7e71b6e80 100644 --- a/nodedb/src/control/planner/calvin/submit/routed.rs +++ b/nodedb/src/control/planner/calvin/submit/routed.rs @@ -204,9 +204,16 @@ pub async fn submit_calvin_routed( }), // The submit went out and its answer was lost. The leader can have // sequenced it, so its outcome is unknown. - Err(ClusterError::Unanswered { .. }) => Err(Error::DeadlineExceeded { - request_id: crate::types::RequestId::new(0), - }), + Err(error @ ClusterError::Unanswered { .. }) => { + tracing::warn!( + leader, + %error, + "calvin-submit reached its deadline with no answer from the sequencer leader" + ); + Err(Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + }) + } Err(e) => Err(Error::Internal { detail: format!("calvin-submit RPC to sequencer leader node {leader} failed: {e}"), }), diff --git a/nodedb/src/control/planner/calvin/tx_class/mod.rs b/nodedb/src/control/planner/calvin/tx_class/mod.rs index a355244f9..1aa9dc862 100644 --- a/nodedb/src/control/planner/calvin/tx_class/mod.rs +++ b/nodedb/src/control/planner/calvin/tx_class/mod.rs @@ -6,20 +6,25 @@ //! task slice: the per-engine write set (`EngineKeySet` — document / vector //! surrogates, KV raw keys, graph-edge identity + routing homes, array tile //! vShards) plus the -//! msgpack-encoded plans. Four builders, split by shape (static vs -//! dependent-read) and participant floor (strict multi-vshard vs the +//! msgpack-encoded plans. The builders split by shape (static, predicted, +//! read-dependent) and participant floor (strict multi-vshard vs the //! single-vshard opt-in): //! //! - [`build_static_tx_class`] / [`build_single_vshard_tx_class`] — every //! write key is known upfront. -//! - [`build_dependent_tx_class`] / [`build_single_vshard_dependent_tx_class`] +//! - [`build_predicted_tx_class`] / [`build_single_vshard_predicted_tx_class`] //! — the OLLP collection's write set comes from reconnaissance-predicted //! surrogates; all other tasks use static extraction. +//! - [`build_read_dependent_tx_class`] — a cross-shard write whose values +//! depend on rows of another vShard. Its passive vShards broadcast those +//! rows to its active vShards through their data-group logs. -pub mod dependent_builder; +pub mod predicted_builder; +pub mod read_dependent_builder; pub mod shared; pub mod static_builder; pub mod write_keys; -pub use dependent_builder::{build_dependent_tx_class, build_single_vshard_dependent_tx_class}; +pub use predicted_builder::{build_predicted_tx_class, build_single_vshard_predicted_tx_class}; +pub use read_dependent_builder::{PassiveReads, build_read_dependent_tx_class}; pub use static_builder::{build_single_vshard_tx_class, build_static_tx_class}; diff --git a/nodedb/src/control/planner/calvin/tx_class/dependent_builder.rs b/nodedb/src/control/planner/calvin/tx_class/predicted_builder.rs similarity index 88% rename from nodedb/src/control/planner/calvin/tx_class/dependent_builder.rs rename to nodedb/src/control/planner/calvin/tx_class/predicted_builder.rs index 87200af9e..46f86d2c2 100644 --- a/nodedb/src/control/planner/calvin/tx_class/dependent_builder.rs +++ b/nodedb/src/control/planner/calvin/tx_class/predicted_builder.rs @@ -1,15 +1,14 @@ // SPDX-License-Identifier: BUSL-1.1 -//! `TxClass` construction for a dependent-read (OLLP) transaction: the OLLP +//! `TxClass` construction for a predicted (OLLP) transaction: the OLLP //! collection's write set comes from reconnaissance-predicted surrogates; all //! other tasks use static surrogate extraction. //! -//! Despite the "dependent" naming (shared with the OLLP reconnaissance -//! terminology), the `TxClass` built here carries `dependent_reads: None` — -//! it is NOT a Calvin dependent-read-barrier transaction (see -//! [`TxClass::new_dependent`]; that is a distinct mechanism for passive -//! vshards to broadcast reads to active participants before they write). This -//! builder is write-set-identity construction only, exactly like +//! The `TxClass` built here carries `dependent_reads: None`. Its participant +//! verifies the prediction against its own rows at stage time. A transaction +//! whose writes depend on rows of another vShard is built by +//! [`super::read_dependent_builder`] instead. This builder is +//! write-set-identity construction only, exactly like //! `build_static_tx_class`, sourced from a pre-exec-scan surrogate //! prediction instead of statically-known plan fields. @@ -25,7 +24,7 @@ use super::write_keys::{WriteKeys, add_plan_write_keys, task_write_keys}; use crate::control::planner::calvin::is_dependent_predicate; use crate::control::planner::calvin::write_class::is_write_plan; -/// Build a **multi-vshard** `TxClass` for a dependent-read (OLLP) transaction. +/// Build a **multi-vshard** `TxClass` for a predicted (OLLP) transaction. /// /// For `BulkUpdate`/`BulkDelete` plans that have `ollp_predicted_surrogates` /// set, the OLLP collection's write set is built from `predicted_surrogates`. @@ -35,7 +34,7 @@ use crate::control::planner::calvin::write_class::is_write_plan; /// produce a valid multi-vshard `TxClass`. A write set that collapses to a /// single vshard is rejected (`SingleVshardTxn`) — for the legitimate /// contended single-collection predicate write, use -/// [`build_single_vshard_dependent_tx_class`]. +/// [`build_single_vshard_predicted_tx_class`]. /// /// `reads` is the neutral session read-set, projected onto the `TxClass`'s /// routing/identity `read_set` (collection-homed) so read shards are enumerated @@ -46,14 +45,14 @@ use crate::control::planner::calvin::write_class::is_write_plan; /// Returns `Ok(None)` when the batch turns out to write nothing at all — the /// predicate matched no rows and no other task in the batch carries a write. /// Returns `Err` if encoding fails or the resulting TxClass is invalid. -pub fn build_dependent_tx_class( +pub fn build_predicted_tx_class( tasks: &[PhysicalTask], tenant_id: TenantId, collection: &str, predicted_surrogates: &[u32], reads: &[ReadSetEntry], ) -> crate::Result> { - build_dependent_tx_class_impl( + build_predicted_tx_class_impl( tasks, tenant_id, collection, @@ -63,7 +62,7 @@ pub fn build_dependent_tx_class( ) } -/// Build a `TxClass` for a dependent-read (OLLP) transaction that is permitted +/// Build a `TxClass` for a predicted (OLLP) transaction that is permitted /// to resolve to a **single vshard**. /// /// Used only by the contended single-collection predicate write routing path @@ -71,19 +70,19 @@ pub fn build_dependent_tx_class( /// gate returned `RouteToCalvin` because a pending commit holds a key in the /// predicate's range, so the write must sequence through the deterministic /// scheduler to serialize on the SAME shared per-vShard `LockManager` the -/// holder is on. Identical extraction to [`build_dependent_tx_class`]; only +/// holder is on. Identical extraction to [`build_predicted_tx_class`]; only /// the participant floor differs (via [`TxClass::new_single_vshard`] — the /// SAME opt-in the point-write path uses, since this `TxClass` shape is a /// plain write-set-identity construction like the static builder, not a /// Calvin dependent-read-barrier transaction). -pub fn build_single_vshard_dependent_tx_class( +pub fn build_single_vshard_predicted_tx_class( tasks: &[PhysicalTask], tenant_id: TenantId, collection: &str, predicted_surrogates: &[u32], reads: &[ReadSetEntry], ) -> crate::Result> { - build_dependent_tx_class_impl( + build_predicted_tx_class_impl( tasks, tenant_id, collection, @@ -93,12 +92,12 @@ pub fn build_single_vshard_dependent_tx_class( ) } -/// Shared body for the dependent builders. `allow_single_vshard` selects +/// Shared body for the predicted builders. `allow_single_vshard` selects /// between [`TxClass::new`] (multi-vshard, `>=2` floor) and /// [`TxClass::new_single_vshard`] (single-vshard opt-in) — the same pair /// `build_static_tx_class_impl` selects between; this builder only differs in /// how the write set's surrogate identity is sourced. -fn build_dependent_tx_class_impl( +fn build_predicted_tx_class_impl( tasks: &[PhysicalTask], tenant_id: TenantId, collection: &str, @@ -142,7 +141,7 @@ fn build_dependent_tx_class_impl( let plans: Vec<&PhysicalPlan> = tasks.iter().map(|t| &t.plan).collect(); let plans_bytes = zerompk::to_msgpack_vec(&plans).map_err(|e| Error::Serialization { format: "msgpack".to_owned(), - detail: format!("failed to encode PhysicalPlan vec for Calvin dependent TxClass: {e}"), + detail: format!("failed to encode PhysicalPlan vec for Calvin predicted TxClass: {e}"), })?; // versioned_reads carries the write-versioned OCC validation set, populated @@ -172,7 +171,7 @@ fn build_dependent_tx_class_impl( ) }; result.map(Some).map_err(|e| Error::BadRequest { - detail: format!("invalid dependent TxClass: {e}"), + detail: format!("invalid predicted TxClass: {e}"), }) } @@ -228,16 +227,16 @@ mod tests { .as_u32(); // Strict builder rejects the single-vshard write set. - let strict = build_dependent_tx_class(&tasks, TenantId::new(1), "users", &[7, 8], &[]); + let strict = build_predicted_tx_class(&tasks, TenantId::new(1), "users", &[7, 8], &[]); assert!( matches!(strict, Err(crate::Error::BadRequest { .. })), - "strict dependent builder must reject single-vshard write set" + "strict predicted builder must reject single-vshard write set" ); // Single-vshard builder accepts it, with exactly one participating vshard. let tx = - build_single_vshard_dependent_tx_class(&tasks, TenantId::new(1), "users", &[7, 8], &[]) - .expect("single-vshard dependent TxClass accepted") + build_single_vshard_predicted_tx_class(&tasks, TenantId::new(1), "users", &[7, 8], &[]) + .expect("single-vshard predicted TxClass accepted") .expect("non-empty write set builds a TxClass"); assert_eq!(tx.participating_vshards().len(), 1); assert_eq!(tx.participating_vshards()[0].as_u32(), want_vshard); @@ -250,7 +249,7 @@ mod tests { // error the retry loop will mistake for predicate drift. let tasks = vec![bulk_delete_task("users")]; let built = - build_single_vshard_dependent_tx_class(&tasks, TenantId::new(1), "users", &[], &[]) + build_single_vshard_predicted_tx_class(&tasks, TenantId::new(1), "users", &[], &[]) .expect("zero-match predicate is not an error"); assert!(built.is_none(), "zero-match predicate builds no TxClass"); } @@ -284,14 +283,14 @@ mod tests { use crate::control::cluster::calvin::scheduler::lock_manager::TxnId; use crate::control::cluster::calvin::scheduler::{AcquireOutcome, LockManager}; - let predicate = build_single_vshard_dependent_tx_class( + let predicate = build_single_vshard_predicted_tx_class( &[bulk_delete_task("users")], TenantId::new(1), "users", &[7], &[], ) - .expect("dependent TxClass") + .expect("predicted TxClass") .expect("non-empty write set"); let insert_task = PhysicalTask { tenant_id: TenantId::new(1), diff --git a/nodedb/src/control/planner/calvin/tx_class/read_dependent_builder.rs b/nodedb/src/control/planner/calvin/tx_class/read_dependent_builder.rs new file mode 100644 index 000000000..0db6b3899 --- /dev/null +++ b/nodedb/src/control/planner/calvin/tx_class/read_dependent_builder.rs @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `TxClass` construction for a read-dependent transaction: a cross-shard +//! write whose values depend on rows of another vShard. +//! +//! The coordinator reads the rows first and plans the writes from what it +//! read. The class carries a [`DependentReadSpec`]: the rows each passive +//! vShard reads again under the transaction's locks, and the values the +//! coordinator read. Each active vShard waits at its dependent-read barrier +//! for every passive vShard's values, through its own data-group log, and +//! votes `PredictionDrift` when one differs. The coordinator then reads +//! again and resubmits. +//! +//! Every passive key set also enters the read set, so the passive vShard +//! locks the rows it reads shared until the transaction's verdict. + +use std::collections::BTreeMap; + +use nodedb_cluster::calvin::types::{ + DependentReadSpec, EngineKeySet, PassiveReadKey, PassiveReadKeyId, ReadWriteSet, TxClass, + VersionedReadSet, +}; +use nodedb_physical::physical_plan::PhysicalPlan; +use nodedb_physical::physical_task::PhysicalTask; +use nodedb_types::{DatabaseId, TenantId}; + +use super::write_keys::task_write_keys; +use crate::Error; + +/// The rows a read-dependent transaction reads on other vShards, and the +/// values its coordinator read for them. +#[derive(Debug, Clone, Default)] +pub struct PassiveReads { + /// Each passive vShard and the key sets it reads. + pub keys: BTreeMap>, + /// The stored bytes the coordinator read for each row, `None` for an + /// absent row. + pub expected: BTreeMap>>, +} + +/// Build a multi-vShard `TxClass` for `tasks`, whose values depend on +/// `reads`. +/// +/// The write set comes from the tasks' static write keys, as +/// [`super::build_static_tx_class`] derives it. The write set spans two or +/// more vShards. A `PassiveReads` with no key set is refused: the class +/// would carry a barrier that waits for nothing. +pub fn build_read_dependent_tx_class( + tasks: &[PhysicalTask], + tenant_id: TenantId, + reads: PassiveReads, +) -> crate::Result { + let database_id = tasks + .first() + .map_or(DatabaseId::DEFAULT, |task| task.database_id); + if tasks.iter().any(|task| task.database_id != database_id) { + return Err(Error::BadRequest { + detail: "Calvin transaction spans multiple databases".to_owned(), + }); + } + if reads.keys.values().all(Vec::is_empty) { + return Err(Error::Internal { + detail: "a read-dependent Calvin transaction names no row it reads".to_owned(), + }); + } + + let write_set = ReadWriteSet::new(task_write_keys(tasks)?.into_key_sets()); + let read_set = ReadWriteSet::new(reads.keys.values().flatten().cloned().collect()); + let passive_reads = reads + .keys + .into_iter() + .map(|(vshard, key_sets)| { + let keys = key_sets + .into_iter() + .map(|engine_key| PassiveReadKey { engine_key }) + .collect(); + (vshard, keys) + }) + .collect(); + let spec = DependentReadSpec { + passive_reads, + expected: reads.expected, + }; + + let plans: Vec<&PhysicalPlan> = tasks.iter().map(|t| &t.plan).collect(); + let plans_bytes = zerompk::to_msgpack_vec(&plans).map_err(|e| Error::Serialization { + format: "msgpack".to_owned(), + detail: format!("failed to encode PhysicalPlan vec for Calvin read-dependent TxClass: {e}"), + })?; + + TxClass::new_dependent( + read_set, + write_set, + plans_bytes, + tenant_id, + database_id, + spec, + VersionedReadSet::default(), + ) + .map_err(|e| Error::BadRequest { + detail: format!("invalid read-dependent TxClass: {e}"), + }) +} + +#[cfg(test)] +mod tests { + use nodedb_cluster::calvin::types::SortedVec; + use nodedb_physical::physical_plan::KvOp; + use nodedb_types::{QualifiedCollection, RlsWriteCheck, Surrogate}; + + use super::*; + use crate::types::VShardId; + + /// Two collection names whose default-database vShards differ. + fn two_distinct_collections() -> (String, String) { + let first = "read_dep_0".to_owned(); + let second = (1u32..4096) + .map(|i| format!("read_dep_{i}")) + .find(|name| vshard_of(name) != vshard_of(&first)) + .expect("a collection on another vShard"); + (first, second) + } + + fn kv_task(plan: KvOp, collection: &str) -> PhysicalTask { + PhysicalTask { + tenant_id: TenantId::new(1), + vshard_id: nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, collection) + .vshard(), + database_id: DatabaseId::DEFAULT, + plan: PhysicalPlan::Kv(plan), + post_set_op: nodedb_physical::physical_task::PostSetOp::None, + txn_id: None, + } + } + + fn vshard_of(collection: &str) -> u32 { + nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, collection) + .vshard() + .as_u32() + } + + /// The class carries the passive reads and the coordinator's values, + /// locks the passive rows shared, and spans the write vShards. + #[test] + fn a_read_dependent_class_carries_its_passive_reads() { + let (source, dest) = two_distinct_collections(); + let tasks = vec![ + kv_task( + KvOp::Delete { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &source), + keys: vec![b"alice:sword".to_vec()], + rls_write_check: RlsWriteCheck::pending_injection(), + returning: None, + rls_filters: Vec::new(), + provenance: None, + }, + &source, + ), + kv_task( + KvOp::Put { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &dest), + key: b"bob:sword".to_vec(), + value: b"item".to_vec(), + ttl_ms: 0, + surrogate: Surrogate::new(3), + returning: None, + rls_filters: Vec::new(), + provenance: None, + }, + &dest, + ), + ]; + let row = PassiveReadKeyId::kv( + QualifiedCollection::new(DatabaseId::DEFAULT, &source), + b"alice:sword".to_vec(), + ); + let reads = PassiveReads { + keys: BTreeMap::from([( + vshard_of(&source), + vec![EngineKeySet::Kv { + collection: source.clone(), + keys: SortedVec::new(vec![b"alice:sword".to_vec()]), + }], + )]), + expected: BTreeMap::from([(row.clone(), Some(b"item".to_vec()))]), + }; + + let class = + build_read_dependent_tx_class(&tasks, TenantId::new(1), reads).expect("valid class"); + + let spec = class.dependent_reads.as_ref().expect("dependent reads"); + assert_eq!( + spec.passive_reads.keys().copied().collect::>(), + vec![vshard_of(&source)] + ); + assert_eq!(spec.expected.get(&row), Some(&Some(b"item".to_vec()))); + assert_eq!(class.read_set.0.len(), 1); + let mut active = class.active_vshards().expect("active vShards"); + active.sort_unstable(); + let mut expected_active = vec![vshard_of(&source), vshard_of(&dest)]; + expected_active.sort_unstable(); + assert_eq!(active, expected_active); + assert!( + class + .participating_vshards() + .contains(&VShardId::new(vshard_of(&source))) + ); + } + + /// A class that names no passive row is refused. + #[test] + fn a_class_with_no_passive_row_is_refused() { + let (source, _) = two_distinct_collections(); + let tasks = vec![kv_task( + KvOp::Delete { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, &source), + keys: vec![b"k".to_vec()], + rls_write_check: RlsWriteCheck::pending_injection(), + returning: None, + rls_filters: Vec::new(), + provenance: None, + }, + &source, + )]; + assert!( + build_read_dependent_tx_class(&tasks, TenantId::new(1), PassiveReads::default()) + .is_err() + ); + } +} diff --git a/nodedb/src/control/planner/calvin/tx_class/write_keys/crdt.rs b/nodedb/src/control/planner/calvin/tx_class/write_keys/crdt.rs index 743e5805a..9c23a4e7e 100644 --- a/nodedb/src/control/planner/calvin/tx_class/write_keys/crdt.rs +++ b/nodedb/src/control/planner/calvin/tx_class/write_keys/crdt.rs @@ -5,6 +5,13 @@ //! A CRDT document is keyed by its document id, bound or not: a delete //! planned before a concurrent upsert bound the document locks the key the //! upsert locks. A write of the whole collection locks the collection key. +//! +//! A constraint install or drop changes no row. It replaces the validator +//! rules a CRDT delta passes at its log apply, and each delta carries the +//! constraint version it was admitted against: the apply fences a delta whose +//! version the replica has not installed yet. So the install needs no +//! exclusion from row writers, and it takes its collection `Intent` like one. +//! It still orders against a writer of the whole collection. #![deny(clippy::wildcard_enum_match_arm)] @@ -56,9 +63,10 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &CrdtOp) -> crate::Result<()> { document_id, .. } => keys.row_id(collection.as_str(), document_id), - CrdtOp::ImportSnapshot { collection, .. } - | CrdtOp::SetConstraints { collection, .. } - | CrdtOp::DropConstraints { collection, .. } => keys.whole_collection(collection.as_str()), + CrdtOp::ImportSnapshot { collection, .. } => keys.whole_collection(collection.as_str()), + CrdtOp::SetConstraints { collection, .. } | CrdtOp::DropConstraints { collection, .. } => { + keys.rows(collection.as_str(), []) + } CrdtOp::Read { .. } | CrdtOp::ReadConstraints { .. } | CrdtOp::SetPolicy { .. } @@ -73,3 +81,45 @@ pub(super) fn add_keys(keys: &mut WriteKeys, op: &CrdtOp) -> crate::Result<()> { } Ok(()) } + +#[cfg(test)] +mod tests { + use nodedb_types::{DatabaseId, QualifiedCollection}; + + use super::*; + use crate::control::cluster::calvin::scheduler::driver::helpers::expand_write_key_sets; + use crate::control::cluster::calvin::scheduler::lock::{LockKey, LockMode}; + + fn locks(op: &CrdtOp) -> Vec<(LockKey, LockMode)> { + let mut keys = WriteKeys::default(); + add_keys(&mut keys, op).expect("a write"); + expand_write_key_sets(&keys.into_key_sets()) + .into_iter() + .collect() + } + + fn collection_key() -> LockKey { + LockKey::Collection { + collection: std::sync::Arc::from("docs"), + } + } + + /// A constraint install takes its collection `Intent` alone, so it runs + /// beside a row writer of the collection. + #[test] + fn a_constraint_install_takes_its_collection_intent() { + let collection = QualifiedCollection::new(DatabaseId::DEFAULT, "docs"); + let set = CrdtOp::SetConstraints { + collection: collection.clone(), + constraint_version: 1, + constraints: Vec::new(), + }; + let drop = CrdtOp::DropConstraints { + collection, + constraint_version: 2, + }; + for op in [set, drop] { + assert_eq!(locks(&op), vec![(collection_key(), LockMode::Intent)]); + } + } +} diff --git a/nodedb/src/control/security/catalog/bootstrap_tables.rs b/nodedb/src/control/security/catalog/bootstrap_tables.rs index a99c5290d..15c0953ee 100644 --- a/nodedb/src/control/security/catalog/bootstrap_tables.rs +++ b/nodedb/src/control/security/catalog/bootstrap_tables.rs @@ -79,6 +79,7 @@ pub(super) const BOOTSTRAP_TABLES: &[BootstrapTable] = bootstrap_tables![ "tenant_group_marks" => super::tenant_group_marks::TENANT_GROUP_MARKS, "tenant_group_restore_marks" => super::tenant_group_marks::TENANT_GROUP_RESTORE_MARKS, "calvin_applied" => super::calvin_applied::CALVIN_APPLIED, + "calvin_barrier_logs" => super::calvin_barrier_logs::CALVIN_BARRIER_LOGS, "calvin_base" => super::calvin_base::CALVIN_BASE, "calvin_sequencer_install" => super::calvin_base::CALVIN_SEQUENCER_INSTALL, "redo_snapshot_owed" => super::redo_snapshot_owed::REDO_SNAPSHOT_OWED, diff --git a/nodedb/src/control/security/catalog/calvin_barrier_logs.rs b/nodedb/src/control/security/catalog/calvin_barrier_logs.rs new file mode 100644 index 000000000..9be335917 --- /dev/null +++ b/nodedb/src/control/security/catalog/calvin_barrier_logs.rs @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Persistent dependent-read barrier logs backing +//! `_system.calvin_barrier_logs`. +//! +//! The data-group apply loop folds each `CalvinReadResult` and +//! `CalvinReadTimeout` entry into the row of its `(vShard, txn)` before the +//! entry counts as applied. Boot resumes Raft delivery above the durable +//! applied floor, so an entry at or below it is never delivered again. Its +//! row is what a barrier of the txn reads after a restart. +//! +//! Each row holds the encoded barrier log of one txn on one vShard. This +//! module stores the bytes and never decodes them. + +use std::collections::BTreeSet; + +use redb::{ReadableDatabase, ReadableTable, TableDefinition}; + +use super::types::{SystemCatalog, catalog_err}; + +/// Table: `(vshard_id, epoch, position)` -> encoded barrier log. +pub(super) const CALVIN_BARRIER_LOGS: TableDefinition<(u32, u64, u32), &[u8]> = + TableDefinition::new("_system.calvin_barrier_logs"); + +/// One stored barrier log. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredBarrierLog { + pub vshard_id: u32, + pub epoch: u64, + pub position: u32, + /// The encoded barrier log. + pub log: Vec, +} + +/// The key range of every row of `vshard_id`. +fn vshard_range(vshard_id: u32) -> std::ops::RangeInclusive<(u32, u64, u32)> { + (vshard_id, 0, 0)..=(vshard_id, u64::MAX, u32::MAX) +} + +/// Append every row of `range` to `out`. +fn collect_rows( + out: &mut Vec, + range: redb::Range<'_, (u32, u64, u32), &'static [u8]>, +) -> crate::Result<()> { + for row in range { + let (key, value) = row.map_err(|e| catalog_err("read calvin_barrier_logs", e))?; + let (vshard_id, epoch, position) = key.value(); + out.push(StoredBarrierLog { + vshard_id, + epoch, + position, + log: value.value().to_vec(), + }); + } + Ok(()) +} + +impl SystemCatalog { + /// Replace the row of `(vshard_id, epoch, position)` with what `fold` + /// returns for the row it holds, in one transaction. A fold that + /// returns `None` keeps the row, and nothing commits. + pub fn fold_calvin_barrier_log( + &self, + vshard_id: u32, + epoch: u64, + position: u32, + fold: impl FnOnce(Option<&[u8]>) -> crate::Result>>, + ) -> crate::Result<()> { + let write_txn = self + .db + .begin_write() + .map_err(|e| catalog_err("fold_calvin_barrier_log txn", e))?; + { + let mut table = write_txn + .open_table(CALVIN_BARRIER_LOGS) + .map_err(|e| catalog_err("open calvin_barrier_logs", e))?; + let key = (vshard_id, epoch, position); + let folded = { + let held = table + .get(key) + .map_err(|e| catalog_err("get calvin_barrier_logs", e))?; + fold(held.as_ref().map(|row| row.value()))? + }; + let Some(folded) = folded else { + return Ok(()); + }; + table + .insert(key, folded.as_slice()) + .map_err(|e| catalog_err("insert calvin_barrier_logs", e))?; + } + write_txn + .commit() + .map_err(|e| catalog_err("commit calvin_barrier_logs", e)) + } + + /// The row of `(vshard_id, epoch, position)`, when one is stored. + pub fn load_calvin_barrier_log( + &self, + vshard_id: u32, + epoch: u64, + position: u32, + ) -> crate::Result>> { + let read_txn = self + .db + .begin_read() + .map_err(|e| catalog_err("load_calvin_barrier_log read txn", e))?; + let table = read_txn + .open_table(CALVIN_BARRIER_LOGS) + .map_err(|e| catalog_err("open calvin_barrier_logs", e))?; + let row = table + .get((vshard_id, epoch, position)) + .map_err(|e| catalog_err("get calvin_barrier_logs", e))?; + Ok(row.map(|row| row.value().to_vec())) + } + + /// Every stored row of the vShards in `vshards`, or of every vShard + /// when `vshards` is `None`, in key order. + pub fn load_calvin_barrier_logs( + &self, + vshards: Option<&BTreeSet>, + ) -> crate::Result> { + let read_txn = self + .db + .begin_read() + .map_err(|e| catalog_err("load_calvin_barrier_logs read txn", e))?; + let table = read_txn + .open_table(CALVIN_BARRIER_LOGS) + .map_err(|e| catalog_err("open calvin_barrier_logs", e))?; + let mut out = Vec::new(); + match vshards { + None => collect_rows( + &mut out, + table + .iter() + .map_err(|e| catalog_err("iter calvin_barrier_logs", e))?, + )?, + Some(vshards) => { + for vshard_id in vshards { + collect_rows( + &mut out, + table + .range(vshard_range(*vshard_id)) + .map_err(|e| catalog_err("range calvin_barrier_logs", e))?, + )?; + } + } + } + Ok(out) + } + + /// Remove the rows of `keys`, each `(vshard_id, epoch, position)`, in + /// one transaction. A key with no row changes nothing. + pub fn remove_calvin_barrier_logs(&self, keys: &[(u32, u64, u32)]) -> crate::Result<()> { + if keys.is_empty() { + return Ok(()); + } + let write_txn = self + .db + .begin_write() + .map_err(|e| catalog_err("remove_calvin_barrier_logs txn", e))?; + { + let mut table = write_txn + .open_table(CALVIN_BARRIER_LOGS) + .map_err(|e| catalog_err("open calvin_barrier_logs", e))?; + for key in keys { + table + .remove(*key) + .map_err(|e| catalog_err("remove calvin_barrier_logs", e))?; + } + } + write_txn + .commit() + .map_err(|e| catalog_err("commit calvin_barrier_logs", e)) + } + + /// Replace every row of the vShards in `vshards` with `rows`, in one + /// transaction. A data-group snapshot install brings the rows the + /// builder held for the group's vShards: they replace this node's. + pub fn replace_calvin_barrier_logs( + &self, + vshards: &BTreeSet, + rows: &[StoredBarrierLog], + ) -> crate::Result<()> { + let write_txn = self + .db + .begin_write() + .map_err(|e| catalog_err("replace_calvin_barrier_logs txn", e))?; + { + let mut table = write_txn + .open_table(CALVIN_BARRIER_LOGS) + .map_err(|e| catalog_err("open calvin_barrier_logs", e))?; + for vshard_id in vshards { + table + .retain_in(vshard_range(*vshard_id), |_, _| false) + .map_err(|e| catalog_err("clear calvin_barrier_logs", e))?; + } + for row in rows { + table + .insert((row.vshard_id, row.epoch, row.position), row.log.as_slice()) + .map_err(|e| catalog_err("insert calvin_barrier_logs", e))?; + } + } + write_txn + .commit() + .map_err(|e| catalog_err("commit calvin_barrier_logs", e)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn row(vshard_id: u32, epoch: u64, position: u32, log: &[u8]) -> StoredBarrierLog { + StoredBarrierLog { + vshard_id, + epoch, + position, + log: log.to_vec(), + } + } + + fn put(catalog: &SystemCatalog, row: &StoredBarrierLog) { + let bytes = row.log.clone(); + catalog + .fold_calvin_barrier_log(row.vshard_id, row.epoch, row.position, |_| Ok(Some(bytes))) + .expect("fold"); + } + + /// A fold sees the row it replaces, and the row reloads. + #[test] + fn a_fold_sees_the_held_row() { + let catalog = SystemCatalog::open_in_memory().expect("catalog"); + put(&catalog, &row(3, 7, 1, b"a")); + catalog + .fold_calvin_barrier_log(3, 7, 1, |held| { + assert_eq!(held, Some(b"a".as_slice())); + Ok(Some(b"ab".to_vec())) + }) + .expect("fold"); + assert_eq!( + catalog.load_calvin_barrier_log(3, 7, 1).expect("load"), + Some(b"ab".to_vec()) + ); + assert_eq!( + catalog.load_calvin_barrier_log(3, 7, 2).expect("load"), + None + ); + // A fold that keeps the row commits nothing. + catalog + .fold_calvin_barrier_log(3, 7, 1, |_| Ok(None)) + .expect("fold"); + catalog + .fold_calvin_barrier_log(3, 7, 2, |_| Ok(None)) + .expect("fold"); + assert_eq!( + catalog.load_calvin_barrier_log(3, 7, 1).expect("load"), + Some(b"ab".to_vec()) + ); + assert_eq!( + catalog.load_calvin_barrier_log(3, 7, 2).expect("load"), + None + ); + } + + /// A load of some vShards returns their rows alone, in key order. + #[test] + fn rows_load_per_vshard_in_key_order() { + let catalog = SystemCatalog::open_in_memory().expect("catalog"); + put(&catalog, &row(4, 2, 0, b"y")); + put(&catalog, &row(3, 9, 0, b"x")); + put(&catalog, &row(3, 1, 5, b"w")); + let all = catalog.load_calvin_barrier_logs(None).expect("load"); + assert_eq!( + all, + vec![row(3, 1, 5, b"w"), row(3, 9, 0, b"x"), row(4, 2, 0, b"y")] + ); + let only = catalog + .load_calvin_barrier_logs(Some(&BTreeSet::from([4]))) + .expect("load"); + assert_eq!(only, vec![row(4, 2, 0, b"y")]); + } + + /// A replace drops every row of the named vShards and keeps the rest. + #[test] + fn a_replace_swaps_the_named_vshards_rows() { + let catalog = SystemCatalog::open_in_memory().expect("catalog"); + put(&catalog, &row(3, 1, 0, b"old")); + put(&catalog, &row(4, 1, 0, b"kept")); + catalog + .replace_calvin_barrier_logs(&BTreeSet::from([3]), &[row(3, 2, 0, b"new")]) + .expect("replace"); + let all = catalog.load_calvin_barrier_logs(None).expect("load"); + assert_eq!(all, vec![row(3, 2, 0, b"new"), row(4, 1, 0, b"kept")]); + catalog + .remove_calvin_barrier_logs(&[(3, 2, 0), (9, 9, 9)]) + .expect("remove"); + let all = catalog.load_calvin_barrier_logs(None).expect("load"); + assert_eq!(all, vec![row(4, 1, 0, b"kept")]); + } +} diff --git a/nodedb/src/control/security/catalog/mod.rs b/nodedb/src/control/security/catalog/mod.rs index 66b8de550..80ba2fd62 100644 --- a/nodedb/src/control/security/catalog/mod.rs +++ b/nodedb/src/control/security/catalog/mod.rs @@ -9,6 +9,7 @@ pub mod backup_schedule_marks; pub mod blacklist; pub mod bootstrap_tables; pub mod calvin_applied; +pub mod calvin_barrier_logs; pub mod calvin_base; pub mod change_streams; pub mod checkpoint; diff --git a/nodedb/src/control/security/catalog/replicated_image.rs b/nodedb/src/control/security/catalog/replicated_image.rs index 8dadcd828..5d611acb1 100644 --- a/nodedb/src/control/security/catalog/replicated_image.rs +++ b/nodedb/src/control/security/catalog/replicated_image.rs @@ -23,6 +23,8 @@ //! - `tenant_group_marks`, `tenant_group_restore_marks`: data-group write //! marks, carried by data-group snapshots. //! - `calvin_applied`: this node's Calvin apply ledger. +//! - `calvin_barrier_logs`: this node's dependent-read barrier logs, +//! carried by data-group snapshots. //! - `redo_snapshot_owed`: the data-group snapshot installs this node's //! replicas owe. //! - `l2_cleanup_queue`, `pending_reclaim`, `pending_history_compaction`: diff --git a/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/dispatch.rs index 4863e4a11..24cbc8084 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/dispatch.rs @@ -190,7 +190,7 @@ async fn dispatch_autocommit( /// Map a dispatch error to the client-facing error. A Data-Plane verdict /// arrives here as `Error::DataPlane` on the replicated route, and it renders /// the same way the local route's error status does. -fn error_to_ddl(error: &crate::Error) -> DdlError { +pub(crate) fn error_to_ddl(error: &crate::Error) -> DdlError { match error { crate::Error::DataPlane(code) => data_plane_error(Some(code.clone())), other => { diff --git a/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/mod.rs b/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/mod.rs index 2bb312865..972b75cc8 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/mod.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/kv_atomic/mod.rs @@ -19,6 +19,6 @@ pub mod dispatch; pub mod handlers; pub(crate) use dispatch::{ - dispatch_and_respond, parse_function_args, single_text_col, split_args, unquote, + dispatch_and_respond, error_to_ddl, parse_function_args, single_text_col, split_args, unquote, }; pub use handlers::{kv_cas, kv_getset, kv_incr, kv_incr_float}; diff --git a/nodedb/src/control/server/shared/ddl/neutral/mod.rs b/nodedb/src/control/server/shared/ddl/neutral/mod.rs index a7094a850..171977a0d 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/mod.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/mod.rs @@ -83,6 +83,7 @@ pub mod timeseries; pub mod topic; pub mod topic_subscribe; pub mod transfer; +mod transfer_cross_shard; pub mod tree_ops; pub mod trigger; pub mod typeguard; diff --git a/nodedb/src/control/server/shared/ddl/neutral/transfer.rs b/nodedb/src/control/server/shared/ddl/neutral/transfer.rs index f09f1555f..90c5cbf40 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/transfer.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/transfer.rs @@ -17,6 +17,8 @@ //! //! Both dispatch to the Data Plane as dedicated KvOp variants. The entire //! read-validate-write executes in a single TPC core pass — no TOCTOU race. +//! A `TRANSFER_ITEM` between collections on two vShards runs as one +//! read-dependent Calvin transaction instead (see `transfer_cross_shard`). use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::shared::session::DmlTxnCtx; @@ -134,22 +136,13 @@ pub async fn transfer_item( let source_owner = unquote(&args[3]); let dest_owner = unquote(&args[4]); - // Cross-collection transfers must be on the same vshard. - // Validate this upfront to prevent silent failures. + // A move between collections on two vShards runs as one read-dependent + // Calvin transaction. A move within one vShard runs on its core. let vshard_src = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, &source_collection).vshard(); let vshard_dst = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, &dest_collection).vshard(); - if source_collection != dest_collection && vshard_src != vshard_dst { - return Err(ddl_err( - "0A000", - format!( - "TRANSFER_ITEM: cross-shard transfer not supported \ - (source '{}' and dest '{}' map to different vShards)", - source_collection, dest_collection - ), - )); - } + let cross_shard = vshard_src != vshard_dst; let item_key = format!("{source_owner}:{item_id}"); let dest_key = format!("{dest_owner}:{item_id}"); @@ -187,6 +180,18 @@ pub async fn transfer_item( dest_rls_write_check: nodedb_types::RlsWriteCheck::pending_injection(), }); + if cross_shard { + return super::transfer_cross_shard::transfer_item_across_shards( + state, + identity, + txn_ctx, + plan, + [source_collection.as_str(), dest_collection.as_str()], + vshard_src, + ) + .await; + } + dispatch_and_respond( state, identity, diff --git a/nodedb/src/control/server/shared/ddl/neutral/transfer_cross_shard.rs b/nodedb/src/control/server/shared/ddl/neutral/transfer_cross_shard.rs new file mode 100644 index 000000000..713b07f3b --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/transfer_cross_shard.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `SELECT TRANSFER_ITEM(...)` between collections on two vShards. +//! +//! The move runs as one read-dependent Calvin transaction (see +//! `control::planner::calvin::read_dependent`): the destination write takes +//! the bytes the source vShard holds under the transaction's locks, and both +//! halves commit or neither does. The handler applies the gates a +//! same-vShard move passes: both collections' grants, the row-level +//! security injection, the clone-write check, and the descriptor write lease +//! held until the move finishes. +//! +//! Inside a transaction block the move is refused: its two halves cannot +//! stage into one vShard's overlay. + +use nodedb_physical::physical_plan::{KvOp, PhysicalPlan}; +use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; + +use crate::control::planner::calvin::read_dependent::{ + CrossShardItemMove, move_item_across_shards, +}; +use crate::control::security::audit::ArcAuditEmitter; +use crate::control::security::identity::{AuthenticatedIdentity, Permission}; +use crate::control::server::shared::clone_write::{ + CloneCheckedOutcome, InterceptAndAuthorizeParams, intercept_and_authorize, +}; +use crate::control::server::shared::session::{DmlTxnCtx, TransactionState}; +use crate::control::state::SharedState; +use crate::types::{DatabaseId, VShardId}; + +use super::super::result::{DdlError, DdlResult}; +use super::kv_atomic::{error_to_ddl, single_text_col}; +use super::read_gate::CollectionReadGate; + +/// Run `plan`, a `KvOp::TransferItem` between `collections` (source, then +/// destination) whose vShards differ, as one Calvin transaction. +/// `source_vshard` is the source collection's vShard. +pub(super) async fn transfer_item_across_shards( + state: &SharedState, + identity: &AuthenticatedIdentity, + txn_ctx: &DmlTxnCtx<'_>, + mut plan: PhysicalPlan, + collections: [&str; 2], + source_vshard: VShardId, +) -> Result, DdlError> { + if txn_ctx.sessions.transaction_state(txn_ctx.session_id) != TransactionState::Idle { + return Err(error_to_ddl(&crate::Error::CrossShardInExplicitTransaction)); + } + let database_id = DatabaseId::DEFAULT; + let gate = CollectionReadGate::for_request(state, identity, database_id); + for collection in collections { + gate.authorize(collection)?; + gate.authorize_permission(collection, Permission::Write)?; + } + gate.inject_rls(&mut plan)?; + + let emitter = ArcAuditEmitter(std::sync::Arc::clone(&state.audit)); + let checked = intercept_and_authorize(InterceptAndAuthorizeParams { + state, + task: PhysicalTask { + tenant_id: identity.tenant_id, + vshard_id: source_vshard, + database_id, + plan, + post_set_op: PostSetOp::None, + txn_id: None, + }, + identity, + tenant_id: identity.tenant_id, + permissions: &state.permissions, + roles: &state.roles, + emitter: &emitter, + }) + .await + .map_err(|e| error_to_ddl(&e))?; + let checked = match checked { + CloneCheckedOutcome::Handled(response) => { + let text = crate::data::executor::response_codec::decode_payload_to_json( + response.payload.as_ref(), + ); + return Ok(vec![single_text_col("transfer_item", text)]); + } + CloneCheckedOutcome::Proceed(checked) => checked, + }; + // The lease holds every collection's descriptor until the move finished, + // so a drain of either waits for it. + let (authorized, _lease) = checked.into_parts(); + let PhysicalPlan::Kv(KvOp::TransferItem { + source_collection, + dest_collection, + item_key, + dest_key, + surrogate, + source_rls_write_check, + dest_rls_write_check: _, + }) = authorized.into_physical_task().plan + else { + return Err(DdlError::internal( + "TRANSFER_ITEM: the authorized task is not the item move", + )); + }; + let reply = serde_json::json!({ + "item_key": String::from_utf8_lossy(&item_key), + "dest_key": String::from_utf8_lossy(&dest_key), + "source_collection": source_collection.as_str(), + "dest_collection": dest_collection.as_str(), + }); + + move_item_across_shards( + state, + identity, + CrossShardItemMove { + tenant_id: identity.tenant_id, + database_id, + source_collection, + dest_collection, + item_key, + dest_key, + surrogate, + source_rls_write_check, + }, + ) + .await + .map_err(|e| error_to_ddl(&e))?; + + let payload = nodedb_types::json_to_msgpack(&reply) + .map_err(|e| DdlError::internal(format!("TRANSFER_ITEM reply: {e}")))?; + let text = crate::data::executor::response_codec::decode_payload_to_json(&payload); + Ok(vec![single_text_col("transfer_item", text)]) +} diff --git a/nodedb/src/control/server/shared/write_admission/gate.rs b/nodedb/src/control/server/shared/write_admission/gate.rs index af9456998..45c487402 100644 --- a/nodedb/src/control/server/shared/write_admission/gate.rs +++ b/nodedb/src/control/server/shared/write_admission/gate.rs @@ -236,9 +236,30 @@ pub(crate) fn admit_routed( let mut request = plan_admission_keys(shared, target.tenant_id, target.database_id, target.plan)?; request.sequenced &= may_route; - Ok(admit_request(shared, vshard, lock_manager, request)) + let admission = admit_request(shared, vshard, lock_manager, request); + if matches!( + admission, + WriteAdmission::RouteToCalvin | WriteAdmission::Wait(_) + ) && tracing::enabled!(tracing::Level::DEBUG) + { + let plan: String = format!("{:?}", target.plan) + .chars() + .take(PLAN_SUMMARY_LEN) + .collect(); + tracing::debug!( + vshard_id = vshard.as_u32(), + routes = matches!(admission, WriteAdmission::RouteToCalvin), + %plan, + "the write gate routes or holds a contended write" + ); + } + Ok(admission) } +/// How many characters of a contended write's plan the gate's debug line +/// names. +const PLAN_SUMMARY_LEN: usize = 240; + /// Admit the lock request `request` on `vshard`, whose lock table is /// `lock_manager`. pub(crate) fn admit_request( @@ -289,10 +310,20 @@ fn probe( } // A write that waits probes again with the same request. let retry_keys = (!sequenced).then(|| keys.clone()); - let acquired = lock_manager - .lock() - .unwrap_or_else(|p| p.into_inner()) - .try_acquire(txn, keys); + let acquired = { + let mut table = lock_manager.lock().unwrap_or_else(|p| p.into_inner()); + let contention = + tracing::enabled!(tracing::Level::DEBUG).then(|| table.contention(txn, &keys)); + let acquired = table.try_acquire(txn, keys); + if !acquired { + tracing::debug!( + sequenced, + ?contention, + "a write found its keys held at the write gate" + ); + } + acquired + }; match (acquired, retry_keys) { (true, _) => WriteAdmission::FastPath { guard: Some(WriteAdmissionGuard::new( diff --git a/nodedb/src/control/server/shared/write_admission/leader_gate.rs b/nodedb/src/control/server/shared/write_admission/leader_gate.rs index e578b25e5..27a8c5c3a 100644 --- a/nodedb/src/control/server/shared/write_admission/leader_gate.rs +++ b/nodedb/src/control/server/shared/write_admission/leader_gate.rs @@ -181,6 +181,7 @@ fn entry_scope(write: &ReplicatedWrite, database_id: crate::types::DatabaseId) - // Calvin bookkeeping, a backup cut, a topic message, a surrogate // binding and an array schema write no row. ReplicatedWrite::CalvinReadResult { .. } + | ReplicatedWrite::CalvinReadTimeout { .. } | ReplicatedWrite::CutBarrier { .. } | ReplicatedWrite::TopicPublish { .. } | ReplicatedWrite::SurrogateBind { .. } diff --git a/nodedb/src/control/server/shared/write_admission/wait.rs b/nodedb/src/control/server/shared/write_admission/wait.rs index 73cb22eac..b33cc5842 100644 --- a/nodedb/src/control/server/shared/write_admission/wait.rs +++ b/nodedb/src/control/server/shared/write_admission/wait.rs @@ -75,6 +75,15 @@ impl AdmissionWait { )); } if tokio::time::timeout_at(deadline, released).await.is_err() { + let contention = self + .lock_manager + .lock() + .unwrap_or_else(|p| p.into_inner()) + .contention(self.txn, &self.keys); + tracing::warn!( + ?contention, + "a contended write reached its deadline waiting for its keys" + ); return Err(crate::Error::DeadlineExceeded { request_id: RequestId::new(0), }); diff --git a/nodedb/src/control/state/calvin_local.rs b/nodedb/src/control/state/calvin_local.rs index 0d7ee5d29..c612b64e3 100644 --- a/nodedb/src/control/state/calvin_local.rs +++ b/nodedb/src/control/state/calvin_local.rs @@ -10,7 +10,7 @@ use std::sync::{Arc, Mutex, OnceLock}; use crate::control::cluster::calvin::scheduler::lock::HotKeyTable; use crate::control::cluster::calvin::scheduler::lock_manager::{LockManager, TxnId}; use crate::control::cluster::calvin::scheduler::{ - CalvinAppliedLedgers, CalvinInboxes, CaughtUpRegistry, SequencerProposer, + CalvinAppliedLedgers, CalvinInboxes, CalvinReadResults, CaughtUpRegistry, SequencerProposer, }; use crate::control::server::shared::write_admission::AdmissionHolds; @@ -98,6 +98,10 @@ pub struct CalvinLocalState { /// The inbox of each running scheduler: the data-group apply loop tells /// it how each stamped Calvin redo concluded. pub inboxes: CalvinInboxes, + /// Dependent-read barrier events of each vShard that wait for its + /// scheduler: read results and timeouts the data-group apply loop + /// applied before the scheduler took them. + pub read_results: CalvinReadResults, } impl CalvinLocalState { @@ -126,6 +130,7 @@ impl CalvinLocalState { applied: CalvinAppliedLedgers::default(), caught_up: CaughtUpRegistry::default(), inboxes: CalvinInboxes::default(), + read_results: CalvinReadResults::default(), } } } diff --git a/nodedb/src/control/state/methods.rs b/nodedb/src/control/state/methods.rs index ee95a44fe..fb48065f9 100644 --- a/nodedb/src/control/state/methods.rs +++ b/nodedb/src/control/state/methods.rs @@ -43,6 +43,19 @@ impl SharedState { .ok_or_else(|| raft_not_started("the raw async raft proposer")) } + /// The propose phase alone: it returns once the data-group leader holds + /// the entry in its log, with the wait for this node's apply of it. It + /// takes no vShard admission slot. Calvin bookkeeping entries, which no + /// write admission orders, propose through it. + pub(in crate::control) fn async_raft_submit( + &self, + ) -> crate::Result<&Arc> { + self.async_raft_proposer_pair + .get() + .map(|pair| &pair.submit) + .ok_or_else(|| raft_not_started("the async raft submit")) + } + /// Handle for proposing to the metadata Raft group. `start_raft` /// installs it before any listener opens. pub fn metadata_raft_handle( @@ -70,14 +83,19 @@ impl SharedState { }) } - /// Install both Raft proposal handles atomically during cluster startup. + /// Install the Raft proposal handles atomically during cluster startup. pub(in crate::control) fn install_async_raft_proposer_pair( &self, sequenced: Arc, raw: Arc, + submit: Arc, ) -> crate::Result<()> { self.async_raft_proposer_pair - .set(super::proposer_pair::AsyncRaftProposerPair { sequenced, raw }) + .set(super::proposer_pair::AsyncRaftProposerPair { + sequenced, + raw, + submit, + }) .map_err(|_| crate::Error::Internal { detail: "async raft proposer already installed".into(), }) diff --git a/nodedb/src/control/state/proposer_pair.rs b/nodedb/src/control/state/proposer_pair.rs index 5d3f21d75..0d19fcfbd 100644 --- a/nodedb/src/control/state/proposer_pair.rs +++ b/nodedb/src/control/state/proposer_pair.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 -//! The two Raft proposal handles `start_raft` installs together. +//! The Raft proposal handles `start_raft` installs together. use std::sync::Arc; @@ -8,4 +8,6 @@ use std::sync::Arc; pub(super) struct AsyncRaftProposerPair { pub(super) sequenced: Arc, pub(super) raw: Arc, + /// The propose phase both proposers are built from. + pub(super) submit: Arc, } diff --git a/nodedb/src/control/vshard_admission.rs b/nodedb/src/control/vshard_admission.rs index 60375a64f..6092b2fbd 100644 --- a/nodedb/src/control/vshard_admission.rs +++ b/nodedb/src/control/vshard_admission.rs @@ -134,8 +134,14 @@ impl VShardAdmissionSequencer { let queued = self.reserve(slot, vshard_id)?; let mut active = tokio::time::timeout_at(deadline, slot.active.lock()) .await - .map_err(|_| crate::Error::DeadlineExceeded { - request_id: crate::types::RequestId::new(0), + .map_err(|_| { + tracing::warn!( + vshard_id = vshard_id.as_u32(), + "a write waited for its vShard's admission slot until its deadline" + ); + crate::Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + } })?; let proposed = submit().await?; if proposed.at.is_some() { @@ -167,7 +173,7 @@ impl Default for VShardAdmissionSequencer { } /// Install raw and admission-sequenced proposal handles, both built from -/// `submit`, in one atomic set. +/// `submit`, and `submit` itself, in one atomic set. pub(crate) fn install_async_raft_proposer( shared: &SharedState, submit: Arc, @@ -176,9 +182,9 @@ pub(crate) fn install_async_raft_proposer( Arc::clone(&shared.vshard_admission_sequencer), Arc::clone(&submit), ); - let raw = raw_async_raft_proposer(submit); + let raw = raw_async_raft_proposer(Arc::clone(&submit)); let expected_raw = Arc::clone(&raw); - shared.install_async_raft_proposer_pair(sequenced, raw)?; + shared.install_async_raft_proposer_pair(sequenced, raw, submit)?; let installed_raw = shared.raw_async_raft_proposer()?; if !Arc::ptr_eq(installed_raw, &expected_raw) { return Err(crate::Error::Internal { diff --git a/nodedb/src/control/wal_replication/decode/entry.rs b/nodedb/src/control/wal_replication/decode/entry.rs index 4232a36bd..7bfdbb723 100644 --- a/nodedb/src/control/wal_replication/decode/entry.rs +++ b/nodedb/src/control/wal_replication/decode/entry.rs @@ -215,6 +215,10 @@ fn to_physical_plan( detail: "CalvinReadResult reached to_physical_plan (should have been intercepted)" .into(), }), + ReplicatedWrite::CalvinReadTimeout { .. } => Err(crate::Error::Internal { + detail: "CalvinReadTimeout reached to_physical_plan (should have been intercepted)" + .into(), + }), // The apply loop applies these through `transaction_redo`, which stamps // the redo with the entry's Raft coordinates. ReplicatedWrite::TransactionRedo { .. } => Err(crate::Error::Internal { diff --git a/nodedb/src/control/wal_replication/types/replicated_write.rs b/nodedb/src/control/wal_replication/types/replicated_write.rs index 87ae3742f..b63fb766b 100644 --- a/nodedb/src/control/wal_replication/types/replicated_write.rs +++ b/nodedb/src/control/wal_replication/types/replicated_write.rs @@ -1100,6 +1100,16 @@ pub enum ReplicatedWrite { RedoAbandon { stream: RedoStreamId, }, + /// The dependent-read barrier of Calvin txn `(epoch, position)` on this + /// vShard waited past its timeout. The vShard's data-group leader + /// proposes it. Every replica applies it in log order: a read result + /// below it in the log counts, one above it does not. A barrier still + /// missing a read result at this entry votes abort on every replica. + CalvinReadTimeout { + epoch: u64, + position: u32, + tenant_id: u64, + }, } #[cfg(test)] diff --git a/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs b/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs index a59701e9d..ea73dc536 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs @@ -14,7 +14,6 @@ use crate::bridge::envelope::{ErrorCode, Payload, Response, StageVote, Status}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::commit_pending::PendingCommit; use crate::data::executor::handlers::control::calvin_reply::{CalvinReply, CalvinStaging}; -use crate::data::executor::response_codec; use crate::data::executor::task::ExecutionTask; use crate::types::TenantId; use nodedb_physical::physical_plan::PhysicalPlan; @@ -28,14 +27,13 @@ use super::shared::CalvinExecCtx; impl CoreLoop { /// Execute a passive-participant dependent-read Calvin txn. /// - /// Reads each key from the local engine state and returns a - /// msgpack-encoded `Vec<(PassiveReadKeyId, Value)>` as the response - /// payload. The Control Plane scheduler collects these values and - /// proposes a `ReplicatedWrite::CalvinReadResult` entry to the - /// per-vshard Raft group so all replicas see the same read results. - /// - /// `Instant::now()` is intentionally absent here — this is a - /// synchronous Data Plane read with no timer interaction. + /// Reads each key from base storage and returns a zerompk-encoded + /// `Vec<(PassiveReadKeyId, Value)>` as the response payload. The + /// scheduler holds the transaction's locks on these keys, so no other + /// write changes them before the transaction's verdict. A successful + /// read votes commit: a passive participant writes nothing. The scheduler + /// proposes the payload as a `ReplicatedWrite::CalvinReadResult` entry + /// to the data group of every active vShard. pub(in crate::data::executor) fn execute_calvin_execute_passive( &mut self, task: &ExecutionTask, @@ -53,18 +51,21 @@ impl CoreLoop { "calvin execute passive: reading keys" ); + let database_id = task.request.database_id.as_u64(); let mut results: Vec<(PassiveReadKeyId, Value)> = Vec::with_capacity(keys_to_read.len()); - for passive_key in keys_to_read { - // Build a PassiveReadKeyId for each surrogate in the engine key set. - // For this v1 handler the engine key set carries single surrogates per - // key (as specified in the design); we iterate all surrogates to be safe. - let values = self.read_passive_key(tenant_id, &passive_key.engine_key); - results.extend(values); + match self.read_passive_key(database_id, tenant_id, &passive_key.engine_key) { + Ok(values) => results.extend(values), + Err(code) => return self.response_error(task, code), + } } - match response_codec::encode_serde(&results) { - Ok(payload) => self.response_with_payload(task, payload), + match zerompk::to_msgpack_vec(&results) { + Ok(payload) => { + let mut response = self.response_with_payload(task, payload); + response.stage_vote = Some(StageVote::Commit); + response + } Err(e) => self.response_error( task, ErrorCode::Internal { @@ -90,11 +91,10 @@ impl CoreLoop { /// and stages nothing, so no redo entry exists for it. The Control /// Plane scheduler releases locks and re-recons on `OllpRetryRequired`. /// - /// `injected_reads` is retained on the wire for future plan variants that - /// reference resolved read values by `PassiveReadKeyId`. The coordinator - /// bakes the read values into concrete point ops and the predicted - /// surrogate set at recon, so the plans are self-contained and stage - /// byte-identically to the static path. + /// The coordinator bakes the values it read into concrete point ops, so + /// the plans are self-contained and stage byte-identically to the static + /// path. The scheduler compared `injected_reads` with those values before + /// it dispatched this stage, and a mismatch never reaches the core. pub(in crate::data::executor) fn execute_calvin_execute_active( &mut self, task: &ExecutionTask, @@ -219,6 +219,128 @@ mod tests { bulk_delete_plan, doc_value, make_task, point_insert_plan, seed_row, }; + fn kv_keys(collection: &str, keys: &[&[u8]]) -> PassiveReadKey { + PassiveReadKey { + engine_key: nodedb_cluster::calvin::types::EngineKeySet::Kv { + collection: collection.to_owned(), + keys: nodedb_cluster::calvin::types::SortedVec::new( + keys.iter().map(|key| key.to_vec()).collect(), + ), + }, + } + } + + fn decode_reads(response: &Response) -> Vec<(PassiveReadKeyId, Value)> { + zerompk::from_msgpack(response.payload.as_ref()).expect("passive payload decodes") + } + + /// A passive read returns each stored key-value row as its stored bytes + /// and an absent row as `Null`, and votes commit. + #[test] + fn calvin_execute_passive_reads_stored_kv_bytes_and_votes_commit() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + core.kv_engine + .put(crate::engine::kv::KvPutParams { + database_id: DatabaseId::DEFAULT.as_u64(), + tenant_id: 1, + collection: "items", + key: b"alice:sword", + value: b"stored-bytes", + ttl_ms: 0, + now_ms: crate::engine::kv::current_ms(), + surrogate: Surrogate::new(5), + }) + .expect("seed kv row"); + + let task = make_task(); + let resp = core.execute_calvin_execute_passive( + &task, + 1, + 0, + &TenantId::new(1), + &[kv_keys("items", &[b"alice:sword", b"bob:sword"])], + ); + + assert_eq!(resp.status, Status::Ok); + assert_eq!(resp.stage_vote, Some(StageVote::Commit)); + let collection = nodedb_types::QualifiedCollection::from_stored("items".to_owned()); + let reads: BTreeMap = decode_reads(&resp).into_iter().collect(); + assert_eq!( + reads.get(&PassiveReadKeyId::kv( + collection.clone(), + b"alice:sword".to_vec() + )), + Some(&Value::Bytes(b"stored-bytes".to_vec())) + ); + assert_eq!( + reads.get(&PassiveReadKeyId::kv(collection, b"bob:sword".to_vec())), + Some(&Value::Null) + ); + } + + /// A passive read returns a stored document row as its stored bytes. + #[test] + fn calvin_execute_passive_reads_stored_document_bytes() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + seed_row(&mut core, "orders", 11); + let stored = core + .sparse + .get( + DatabaseId::DEFAULT.as_u64(), + 1, + "orders", + &nodedb_types::StorageKey::for_surrogate(Surrogate::new(11)), + ) + .expect("read seeded row") + .expect("seeded row present"); + + let task = make_task(); + let keys = [PassiveReadKey { + engine_key: nodedb_cluster::calvin::types::EngineKeySet::Document { + collection: "orders".to_owned(), + surrogates: nodedb_cluster::calvin::types::SortedVec::new(vec![11]), + }, + }]; + let resp = core.execute_calvin_execute_passive(&task, 1, 0, &TenantId::new(1), &keys); + + assert_eq!(resp.status, Status::Ok); + assert_eq!( + decode_reads(&resp), + vec![( + PassiveReadKeyId::surrogate( + nodedb_types::QualifiedCollection::from_stored("orders".to_owned()), + 11 + ), + Value::Bytes(stored) + )] + ); + } + + /// A key set that names no readable row refuses the read, so the + /// passive participant votes abort instead of reporting a value. + #[test] + fn calvin_execute_passive_refuses_a_lock_only_key_set() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + let task = make_task(); + let keys = [PassiveReadKey { + engine_key: nodedb_cluster::calvin::types::EngineKeySet::Collection { + collection: "orders".to_owned(), + vshards: nodedb_cluster::calvin::types::SortedVec::new(Vec::new()), + }, + }]; + let resp = core.execute_calvin_execute_passive(&task, 1, 0, &TenantId::new(1), &keys); + + assert_eq!(resp.status, Status::Error); + assert_eq!(resp.stage_vote, None); + assert!(matches!( + resp.error_code.as_deref(), + Some(ErrorCode::Unsupported { .. }) + )); + } + /// The dependent-read ACTIVE path STAGES its writes (into `commit_pending` + /// the synthetic overlay) instead of applying them to base directly, so a /// Calvin-committed dependent-read write reaches the WAL as a redo record. diff --git a/nodedb/src/data/executor/handlers/control/calvin/mod.rs b/nodedb/src/data/executor/handlers/control/calvin/mod.rs index fcf783aa7..4f8437fd7 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/mod.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/mod.rs @@ -13,10 +13,10 @@ //! `CoreLoop::execute_calvin_drop` discards it. //! //! - `CoreLoop::execute_calvin_execute_passive`: passive participant for a -//! dependent-read txn. Reads each declared key from the local engine and -//! returns a msgpack-encoded `Vec<(PassiveReadKeyId, Value)>` payload. The -//! Control Plane scheduler proposes a `CalvinReadResult` Raft entry after -//! receiving this response. +//! dependent-read txn. Reads each declared key from base storage and +//! returns a zerompk-encoded `Vec<(PassiveReadKeyId, Value)>` payload with +//! a commit vote. The Control Plane scheduler proposes the payload as a +//! `CalvinReadResult` entry to the data group of every active vShard. //! //! - `CoreLoop::execute_calvin_execute_active`: active participant for a //! dependent-read txn. Stages the physical plans, with the injected read diff --git a/nodedb/src/data/executor/handlers/control/calvin_passive_read.rs b/nodedb/src/data/executor/handlers/control/calvin_passive_read.rs index 1fbb62cbb..831bd828d 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_passive_read.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_passive_read.rs @@ -3,172 +3,106 @@ //! Passive-read helpers for the Calvin dependent-read path. //! //! [`CoreLoop::execute_calvin_execute_passive`] reads each declared key from -//! the local engine to build the `Vec<(PassiveReadKeyId, Value)>` payload the -//! Control Plane scheduler proposes as a `CalvinReadResult` Raft entry. The -//! per-engine lookups and the deterministic key-hashing they rely on live -//! here, split out of `calvin.rs` to keep that file within the size limit. +//! base storage to build the `Vec<(PassiveReadKeyId, Value)>` payload the +//! Control Plane scheduler proposes as a `CalvinReadResult` Raft entry. //! -//! The KV/edge surrogate hashes MUST be deterministic across replicas: the -//! same byte key or `(src, dst)` edge must produce the same `u32` on every -//! node, so `DefaultHasher` (RandomState) is explicitly NOT used. +//! A row's value is its stored bytes as `Value::Bytes`, `Value::Null` when +//! the row is absent. Stored bytes compare exactly, so an active participant +//! matches them against the bytes its coordinator read. //! -//! `EngineKeySet.collection` arrives already database-qualified: the Calvin -//! write-key extraction reads it off the physical plan's own qualified -//! collection field. So every `PassiveReadKeyId` below rebuilds it via -//! `from_stored` rather than re-qualifying. +//! `EngineKeySet.collection` arrives database-qualified: the Calvin key +//! extraction reads it off the physical plan's qualified collection field. +//! The engines key their tables by that same string. use nodedb_physical::physical_plan::meta::PassiveReadKeyId; -use nodedb_types::Value; +use nodedb_types::{QualifiedCollection, StorageKey, Surrogate, Value}; +use crate::bridge::envelope::ErrorCode; use crate::data::executor::core_loop::CoreLoop; +use crate::engine::kv::current_ms; use crate::types::TenantId; impl CoreLoop { - /// Read a single `EngineKeySet` from the local engine state, returning - /// `(PassiveReadKeyId, Value)` pairs. + /// Read every row of `engine_key` from base storage. /// - /// For Document/Vector/Edge engine sets: looks up each surrogate in the - /// sparse engine (document store) and returns the stored value or `Null`. - /// For KV engine sets: looks up each byte key in the KV engine. + /// A passive participant reads document rows by surrogate and key-value + /// rows by key. Every other key set names no row a passive read returns, + /// so it is refused with `Unsupported`. pub(super) fn read_passive_key( &self, + database_id: u64, tenant_id: &TenantId, engine_key: &nodedb_cluster::calvin::types::EngineKeySet, - ) -> Vec<(PassiveReadKeyId, Value)> { + ) -> Result, ErrorCode> { use nodedb_cluster::calvin::types::EngineKeySet; + let tid = tenant_id.as_u64(); match engine_key { EngineKeySet::Document { collection, surrogates, - } - | EngineKeySet::Vector { - collection, - surrogates, } => surrogates .iter() .map(|&surrogate| { - let value = self - .read_surrogate_value(tenant_id, collection, surrogate) - .unwrap_or(Value::Null); - ( - PassiveReadKeyId { - collection: nodedb_types::QualifiedCollection::from_stored( - collection.clone(), + let stored = self + .sparse + .get( + database_id, + tid, + collection, + &StorageKey::for_surrogate(Surrogate::new(surrogate)), + ) + .map_err(|e| ErrorCode::Internal { + detail: format!( + "calvin passive read of {collection} row {surrogate}: {e}" ), + })?; + Ok(( + PassiveReadKeyId::surrogate( + QualifiedCollection::from_stored(collection.clone()), surrogate, - }, - value, - ) + ), + stored_value(stored), + )) }) .collect(), - - EngineKeySet::Kv { collection, keys } => keys - .iter() - .map(|k| { - let value = self - .read_kv_value(tenant_id, collection, k) - .unwrap_or(Value::Null); - // For KV, use key bytes as surrogate placeholder (0 sentinel). - // KV keys don't have surrogates; the PassiveReadKeyId identifies - // the collection and a stable u32 hash of the key. - let key_hash = stable_kv_hash(k); - ( - PassiveReadKeyId { - collection: nodedb_types::QualifiedCollection::from_stored( - collection.clone(), + EngineKeySet::Kv { collection, keys } => { + let now_ms = current_ms(); + Ok(keys + .iter() + .map(|key| { + let stored = self + .kv_engine + .get(database_id, tid, collection, key, now_ms); + ( + PassiveReadKeyId::kv( + QualifiedCollection::from_stored(collection.clone()), + key.clone(), ), - surrogate: key_hash, - }, - value, - ) - }) - .collect(), - - EngineKeySet::Edge { - collection, edges, .. - } => edges - .iter() - .map(|&(src, dst)| { - // Edge reads: use a stable hash of (src, dst) as surrogate. - let edge_hash = stable_edge_hash(src, dst); - ( - PassiveReadKeyId { - collection: nodedb_types::QualifiedCollection::from_stored( - collection.clone(), - ), - surrogate: edge_hash, - }, - Value::Null, // Edge existence read: Null = absent, non-Null = present. - ) - }) - .collect(), - - // An array write is a static write: no dependent transaction - // predicts its cells, so no passive participant reads them. - EngineKeySet::Array { .. } => Vec::new(), - // A whole-collection key and a UNIQUE value key are lock keys - // only: no dependent transaction predicts them. - EngineKeySet::Collection { .. } | EngineKeySet::Unique { .. } => Vec::new(), + stored_value(stored), + ) + }) + .collect()) + } + EngineKeySet::Vector { collection, .. } + | EngineKeySet::Edge { collection, .. } + | EngineKeySet::Array { collection, .. } + | EngineKeySet::Collection { collection, .. } + | EngineKeySet::Unique { collection, .. } => Err(ErrorCode::Unsupported { + detail: format!( + "a passive Calvin read names rows of {collection} by a key set that is \ + neither document surrogates nor key-value keys" + ), + }), } } - - /// Read a single document surrogate from the sparse engine. - /// - /// Returns `None` if the surrogate is not present in this core's partition. - pub(super) fn read_surrogate_value( - &self, - tenant_id: &TenantId, - collection: &str, - surrogate: u32, - ) -> Option { - // In v1 this is a thin stub: the full implementation requires a - // synchronous lookup through the sparse engine's redb B-Tree. - // The engine lookup path is available via `self.engine_state` once - // the Data Plane engine access APIs are wired. For now, return None - // (caller maps None → Null). - let _ = (tenant_id, collection, surrogate); - None - } - - /// Read a single KV entry from the KV engine. - /// - /// Returns `None` if the key is not present. - pub(super) fn read_kv_value( - &self, - tenant_id: &TenantId, - collection: &str, - key: &[u8], - ) -> Option { - let _ = (tenant_id, collection, key); - None - } } -/// Stable, deterministic hash of a KV byte key into a u32 surrogate -/// placeholder for use in `PassiveReadKeyId`. -/// -/// Uses xxhash with a fixed seed to satisfy the determinism contract: -/// the same byte key must produce the same hash on every replica. -/// `DefaultHasher` (RandomState) is explicitly NOT used here. -fn stable_kv_hash(key: &[u8]) -> u32 { - // FNV-1a 32-bit with fixed offset basis — no external dependency needed. - // This is a placeholder; a production implementation would use xxhash-rust - // with a fixed seed once the crate is available in this crate's deps. - const FNV_OFFSET: u32 = 2_166_136_261; - const FNV_PRIME: u32 = 16_777_619; - let mut hash = FNV_OFFSET; - for &byte in key { - hash ^= u32::from(byte); - hash = hash.wrapping_mul(FNV_PRIME); +/// The value a passive read reports for a row: its stored bytes, or `Null` +/// for an absent row. +fn stored_value(stored: Option>) -> Value { + match stored { + Some(bytes) => Value::Bytes(bytes), + None => Value::Null, } - hash -} - -/// Stable, deterministic hash of an edge `(src, dst)` into a u32 surrogate -/// placeholder for `PassiveReadKeyId`. -fn stable_edge_hash(src: u32, dst: u32) -> u32 { - // Combine src and dst with a deterministic mix. - let combined: u64 = (u64::from(src) << 32) | u64::from(dst); - stable_kv_hash(&combined.to_le_bytes()) } diff --git a/nodedb/src/diag/context/mod.rs b/nodedb/src/diag/context/mod.rs index 54cdf3cd0..695e302c4 100644 --- a/nodedb/src/diag/context/mod.rs +++ b/nodedb/src/diag/context/mod.rs @@ -53,7 +53,7 @@ pub(in crate::diag) use quota::{ QuotaRowNotInstalled, QuotaRowWriteFailed, QuotaScopePurgeIncomplete, QuotaScopeReplayAborted, ScopeQuotaNotInstalled, }; -pub(in crate::diag) use raft_apply::RaftEntryReapplied; +pub(in crate::diag) use raft_apply::{CalvinBarrierLogStoreFailed, RaftEntryReapplied}; pub(in crate::diag) use recovery::{ReplayRecordUnapplied, WalArchivalFailedTruncationHeld}; pub(in crate::diag) use redo_stream::{ RedoAbandonGivenUp, RedoSnapshotDebtNotRecorded, RedoStreamLostHere, diff --git a/nodedb/src/diag/context/raft_apply.rs b/nodedb/src/diag/context/raft_apply.rs index 892b76af7..13178ca52 100644 --- a/nodedb/src/diag/context/raft_apply.rs +++ b/nodedb/src/diag/context/raft_apply.rs @@ -40,3 +40,56 @@ impl DomainContext for RaftEntryReapplied { }) } } + +/// The catalog refused a read or write of a vShard's stored dependent-read +/// barrier log. +pub(in crate::diag) struct CalvinBarrierLogStoreFailed<'a> { + pub vshard_id: u32, + /// The txn's `(epoch, position)`. A removal of every row of the vShard + /// names none. + pub txn: Option<(u64, u32)>, + /// What the store did: `save`, `load` or `remove`. + pub op: &'static str, + pub error_class: &'a str, +} + +impl DomainContext for CalvinBarrierLogStoreFailed<'_> { + fn domain_kind(&self) -> &'static str { + "nodedb.calvin_barrier_log_store_failed" + } + + fn grouping_key(&self) -> String { + // One report per vShard and operation: the txn changes per event. + format!( + "calvin_barrier_log_store_failed:{}:{}", + self.op, self.vshard_id + ) + } + + fn to_json(&self) -> Value { + let effect = match self.op { + "save" => { + "the barrier event stays in memory, and every later event of the \ + txn too. The entry does not count as durably applied, so a restart \ + delivers it again from the log" + } + "load" => { + "the scheduler keeps the txn's barrier open and reads the row \ + again on its next pass" + } + _ => { + "the row of a finished txn stays stored. The scheduler's next \ + stall-tick sweep removes it" + } + }; + json!({ + "vshard_id": self.vshard_id, + "txn": self.txn, + "op": self.op, + "error_class": self.error_class, + "effect": effect, + "operator_action": "check the system catalog's disk: free space, I/O errors, \ + and the redb file's permissions", + }) + } +} diff --git a/nodedb/src/diag/mod.rs b/nodedb/src/diag/mod.rs index e2768b9bf..82573cd7b 100644 --- a/nodedb/src/diag/mod.rs +++ b/nodedb/src/diag/mod.rs @@ -11,20 +11,21 @@ mod recording; pub use context::{DATABASE_SCOPE, IlpFlushOutcome, LostResponseWrite, TENANT_SCOPE}; pub use recording::{ IndexRebuildTarget, VectorBuildTarget, batch_insert_without_surrogates, calvin_apply_halted, - calvin_completion_timeout, catalog_apply_orphan_row, collection_purge_row_missing, - columnar_segment_corrupt, consumer_group_offsets_retained, continuous_aggregate_not_applied, - crdt_dead_letter_not_enqueued, crdt_dead_letter_not_restored, crdt_dead_letter_not_stored, - cut_barrier_not_placed, data_plane_core_fail_stopped, data_plane_response_lost, - data_plane_responses_lost, descriptor_lease_not_renewed, entry_kind, fts_index_update_failed, - history_compaction_not_applied, ilp_invalid_utf8_drop, ilp_line_read_drop, - index_rebuild_not_installed, metadata_apply_wedged, orphaned_index_entry_after_delete, - quota_row_invalid, quota_row_undecodable, quota_row_write_failed, quota_scope_purge_incomplete, - quota_scope_replay_aborted, raft_entries_reapplied, raft_entry_reapplied, - redo_abandon_given_up, redo_snapshot_debt_not_recorded, redo_stream_lost_here, - replay_record_unapplied, retention_autowire_orphaned, scope_quota_not_installed, - strict_row_image_unrendered, strict_row_undecodable, synonym_group_not_applied, - timeseries_partition_unreadable, timeseries_row_image_undecodable, vector_build_failed, - vector_builder_disconnected, vector_builder_spawn_failed, vector_index_not_applied, - vector_rebuild_unreadable, wal_archival_failed_truncation_held, write_acked_without_durability, - write_window_held, write_window_leaked, + calvin_barrier_log_store_failed, calvin_completion_timeout, catalog_apply_orphan_row, + collection_purge_row_missing, columnar_segment_corrupt, consumer_group_offsets_retained, + continuous_aggregate_not_applied, crdt_dead_letter_not_enqueued, crdt_dead_letter_not_restored, + crdt_dead_letter_not_stored, cut_barrier_not_placed, data_plane_core_fail_stopped, + data_plane_response_lost, data_plane_responses_lost, descriptor_lease_not_renewed, entry_kind, + fts_index_update_failed, history_compaction_not_applied, ilp_invalid_utf8_drop, + ilp_line_read_drop, index_rebuild_not_installed, metadata_apply_wedged, + orphaned_index_entry_after_delete, quota_row_invalid, quota_row_undecodable, + quota_row_write_failed, quota_scope_purge_incomplete, quota_scope_replay_aborted, + raft_entries_reapplied, raft_entry_reapplied, redo_abandon_given_up, + redo_snapshot_debt_not_recorded, redo_stream_lost_here, replay_record_unapplied, + retention_autowire_orphaned, scope_quota_not_installed, strict_row_image_unrendered, + strict_row_undecodable, synonym_group_not_applied, timeseries_partition_unreadable, + timeseries_row_image_undecodable, vector_build_failed, vector_builder_disconnected, + vector_builder_spawn_failed, vector_index_not_applied, vector_rebuild_unreadable, + wal_archival_failed_truncation_held, write_acked_without_durability, write_window_held, + write_window_leaked, }; diff --git a/nodedb/src/diag/recording/mod.rs b/nodedb/src/diag/recording/mod.rs index 6f1c6ac8c..4a9e24f17 100644 --- a/nodedb/src/diag/recording/mod.rs +++ b/nodedb/src/diag/recording/mod.rs @@ -52,7 +52,9 @@ pub use quota::{ quota_row_invalid, quota_row_undecodable, quota_row_write_failed, quota_scope_purge_incomplete, quota_scope_replay_aborted, scope_quota_not_installed, }; -pub use raft_apply::{raft_entries_reapplied, raft_entry_reapplied}; +pub use raft_apply::{ + calvin_barrier_log_store_failed, raft_entries_reapplied, raft_entry_reapplied, +}; pub use recovery::{ batch_insert_without_surrogates, fts_index_update_failed, orphaned_index_entry_after_delete, replay_record_unapplied, strict_row_undecodable, wal_archival_failed_truncation_held, diff --git a/nodedb/src/diag/recording/raft_apply.rs b/nodedb/src/diag/recording/raft_apply.rs index acf338aa3..8799ddacf 100644 --- a/nodedb/src/diag/recording/raft_apply.rs +++ b/nodedb/src/diag/recording/raft_apply.rs @@ -4,7 +4,9 @@ use std::sync::atomic::{AtomicU64, Ordering}; -use faultbox::{Capture, EventKind}; +use faultbox::{Capture, EventKind, error_chain_of}; + +use super::shared::error_class; use crate::diag::context; @@ -34,3 +36,27 @@ pub fn raft_entry_reapplied(group_id: u64, log_index: u64, highest_applied: u64) .with_backtrace() .emit(); } + +/// Report a catalog error on a vShard's stored dependent-read barrier log. +/// Called from the sites that read and write the rows. `op` names the operation: `save`, `load` or `remove`. +pub fn calvin_barrier_log_store_failed( + vshard_id: u32, + txn: Option<(u64, u32)>, + op: &'static str, + err: &crate::Error, +) { + let class = error_class(err); + let ctx = context::CalvinBarrierLogStoreFailed { + vshard_id, + txn, + op, + error_class: &class, + }; + let _ = Capture::new( + EventKind::Error, + "calvin dependent-read barrier log store failed", + ) + .error_chain(error_chain_of(err)) + .domain(&ctx) + .emit(); +} diff --git a/nodedb/src/error_from.rs b/nodedb/src/error_from.rs index fb0d65b0e..2cc32a820 100644 --- a/nodedb/src/error_from.rs +++ b/nodedb/src/error_from.rs @@ -250,9 +250,15 @@ impl From for Error { descriptor: collection, } } - TypedClusterError::DeadlineExceeded { .. } => Error::DeadlineExceeded { - request_id: crate::types::RequestId::new(0), - }, + TypedClusterError::DeadlineExceeded { elapsed_ms } => { + tracing::warn!( + elapsed_ms, + "a remote node reported the request's deadline exceeded" + ); + Error::DeadlineExceeded { + request_id: crate::types::RequestId::new(0), + } + } // A remote shard's Data-Plane verdict, verbatim: rebuilding the // code keeps the SQLSTATE `error_classify` renders locally. TypedClusterError::DataPlane { code } => Error::DataPlane(code.into()), diff --git a/nodedb/src/types/mod.rs b/nodedb/src/types/mod.rs index bc6d44077..22ba096bb 100644 --- a/nodedb/src/types/mod.rs +++ b/nodedb/src/types/mod.rs @@ -22,4 +22,4 @@ pub use snapshot::{ ArrayCellsBlob, SurrogateBindEntry, TenantDataSnapshot, TsFlushedCollectionBlob, TsFlushedPartitionBlob, }; -pub use snapshot_calvin::{GroupCalvinCut, VShardCalvinState}; +pub use snapshot_calvin::{CutBarrierLog, GroupCalvinCut, VShardCalvinState}; diff --git a/nodedb/src/types/snapshot_calvin.rs b/nodedb/src/types/snapshot_calvin.rs index 2783d91c1..585278355 100644 --- a/nodedb/src/types/snapshot_calvin.rs +++ b/nodedb/src/types/snapshot_calvin.rs @@ -33,6 +33,10 @@ pub struct GroupCalvinCut { /// The applied state of each group vShard whose scheduler runs on the /// builder. pub vshards: Vec, + /// The stored dependent-read barrier log of each unfinished txn of a + /// group vShard. It holds every barrier entry of the group's log at or + /// below the snapshot's index, which the receiver never applies. + pub barrier_logs: Vec, } /// One vShard's applied Calvin positions in a [`GroupCalvinCut`]. @@ -56,3 +60,24 @@ pub struct VShardCalvinState { /// The applied `(epoch, position)` pairs above the watermark. pub tail: Vec<(u64, u32)>, } + +/// One txn's stored dependent-read barrier log in a [`GroupCalvinCut`]. +#[derive( + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +#[msgpack(map)] +pub struct CutBarrierLog { + pub vshard_id: u32, + pub epoch: u64, + pub position: u32, + /// The encoded barrier log. + pub log: Vec, +} diff --git a/nodedb/tests/inproc/cases/calvin_executor_caps.rs b/nodedb/tests/inproc/cases/calvin_executor_caps.rs index f4546864a..b07484c8f 100644 --- a/nodedb/tests/inproc/cases/calvin_executor_caps.rs +++ b/nodedb/tests/inproc/cases/calvin_executor_caps.rs @@ -85,6 +85,7 @@ fn submit_rejects_dependent_read_too_large() { ); m }, + expected: BTreeMap::new(), }); let err = inbox.submit(tx).expect_err("should reject: too large"); @@ -122,6 +123,7 @@ fn submit_accepts_dependent_read_within_byte_limit() { ); m }, + expected: BTreeMap::new(), }); inbox.submit(tx).expect("should accept: exactly at limit"); @@ -154,6 +156,7 @@ fn submit_rejects_dependent_read_fanout_too_wide() { } m }, + expected: BTreeMap::new(), }); let err = inbox.submit(tx).expect_err("should reject: too wide"); @@ -196,6 +199,7 @@ fn submit_accepts_dependent_read_exactly_at_fanout_limit() { } m }, + expected: BTreeMap::new(), }); inbox diff --git a/nodedb/tests/inproc/cases/calvin_executor_dependent_read.rs b/nodedb/tests/inproc/cases/calvin_executor_dependent_read.rs index 0d510760f..c4e2b4a0c 100644 --- a/nodedb/tests/inproc/cases/calvin_executor_dependent_read.rs +++ b/nodedb/tests/inproc/cases/calvin_executor_dependent_read.rs @@ -1,37 +1,28 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Integration tests for the Calvin dependent-read path. -//! -//! Uses mock channels to simulate the passive → active read result flow -//! without a real cluster. +//! Integration tests for the Calvin dependent-read barrier's public surface: +//! the dependent transaction class, the barrier log a vShard folds from its +//! data-group log, and the check of the read values against the values the +//! coordinator read. -use std::collections::BTreeMap; -use std::time::Duration; - -use nodedb::control::cluster::calvin::scheduler::driver::barrier::{ - PendingDependentBarrier, ReadResultEvent, -}; -use nodedb::control::cluster::calvin::scheduler::lock_manager::TxnId; -use nodedb_physical::physical_plan::meta::PassiveReadKeyId; -use nodedb_types::{TenantId, Value}; - -use std::collections::BTreeSet; -use std::time::Instant; +use std::collections::{BTreeMap, BTreeSet}; +use nodedb::control::cluster::calvin::scheduler::driver::barrier::expected_reads_drift; +use nodedb::control::cluster::calvin::scheduler::{BarrierEvent, BarrierLog, BarrierOutcome}; use nodedb_cluster::calvin::types::{ - DependentReadSpec, EngineKeySet, PassiveReadKey, ReadWriteSet, SequencedTxn, SortedVec, - TxClass, VersionedReadSet, + DependentReadSpec, EngineKeySet, PassiveReadKey, ReadWriteSet, SortedVec, TxClass, + VersionedReadSet, }; -use nodedb_types::{DatabaseId, QualifiedCollection}; +use nodedb_physical::physical_plan::meta::PassiveReadKeyId; +use nodedb_types::{DatabaseId, QualifiedCollection, TenantId, Value}; fn two_distinct_collections() -> (String, String) { let mut first: Option<(String, u32)> = None; for i in 0u32..512 { let name = format!("col_{i}"); - let vshard = - nodedb_types::CollectionKey::from_bare(nodedb::types::DatabaseId::DEFAULT, &name) - .vshard() - .as_u32(); + let vshard = nodedb_types::CollectionKey::from_bare(DatabaseId::DEFAULT, &name) + .vshard() + .as_u32(); if let Some((ref fname, fv)) = first { if fv != vshard { return (fname.clone(), name); @@ -43,91 +34,24 @@ fn two_distinct_collections() -> (String, String) { panic!("could not find two distinct-vshard collections in 512 tries"); } -fn make_dependent_txn(passive_vshard: u32) -> SequencedTxn { - let (col_a, col_b) = two_distinct_collections(); - let write_set = ReadWriteSet::new(vec![ - EngineKeySet::Document { - collection: col_a.clone(), - surrogates: SortedVec::new(vec![1]), - }, - EngineKeySet::Document { - collection: col_b.clone(), - surrogates: SortedVec::new(vec![2]), - }, - ]); - let spec = DependentReadSpec { - passive_reads: { - let mut m = BTreeMap::new(); - m.insert( - passive_vshard, - vec![PassiveReadKey { - engine_key: EngineKeySet::Document { - collection: "passive_col".to_owned(), - surrogates: SortedVec::new(vec![42u32]), - }, - }], - ); - m - }, - }; - let tx_class = TxClass::new_dependent( - ReadWriteSet::new(vec![]), - write_set, - vec![], - TenantId::new(1), - spec, - VersionedReadSet::default(), +fn row(collection: &str, surrogate: u32) -> PassiveReadKeyId { + PassiveReadKeyId::surrogate( + QualifiedCollection::new(DatabaseId::DEFAULT, collection), + surrogate, ) - .expect("valid dependent TxClass"); - SequencedTxn { - epoch: 1, - position: 0, - tx_class, - epoch_system_ms: 0, - epoch_vshard_txn_count: 0, - lock_owner: None, - } } -#[test] -fn dependent_barrier_completes_after_passive_delivers() { - let txn = make_dependent_txn(7); - let passive_vshard = 7u32; - - let mut waiting_for = BTreeSet::new(); - waiting_for.insert(passive_vshard); - - let timeout_at = Instant::now() + Duration::from_secs(30); - - let mut barrier = PendingDependentBarrier { - txn: txn.clone(), - lock_owner: TxnId::new(txn.epoch, txn.position), - waiting_for, - received: BTreeMap::new(), - timeout_at, - }; - - assert!(!barrier.is_complete()); - - // Simulate the passive vshard delivering its read result. - let key_id = PassiveReadKeyId { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "passive_col"), - surrogate: 42, - }; - let values = vec![(key_id.clone(), Value::Integer(100))]; - - barrier.waiting_for.remove(&passive_vshard); - barrier.received.insert(passive_vshard, values); - - assert!(barrier.is_complete(), "barrier should be complete now"); - - let injected = barrier.assemble_injected_reads(); - assert_eq!(injected.len(), 1); - assert_eq!(injected.get(&key_id), Some(&Value::Integer(100))); +fn read(passive_vshard: u32, collection: &str, value: &[u8]) -> BarrierEvent { + BarrierEvent::Read { + passive_vshard, + values: vec![(row(collection, 1), Value::Bytes(value.to_vec()))], + } } +/// A dependent class names every passive vShard as a participant, and its +/// write vShards as its active participants. #[test] -fn dependent_barrier_not_complete_with_multiple_passives() { +fn a_dependent_class_participates_its_passive_and_active_vshards() { let (col_a, col_b) = two_distinct_collections(); let write_set = ReadWriteSet::new(vec![ EngineKeySet::Document { @@ -139,136 +63,79 @@ fn dependent_barrier_not_complete_with_multiple_passives() { surrogates: SortedVec::new(vec![2]), }, ]); - + let passive_vshard = 977u32; let spec = DependentReadSpec { - passive_reads: { - let mut m = BTreeMap::new(); - m.insert( - 10u32, - vec![PassiveReadKey { - engine_key: EngineKeySet::Document { - collection: "coll_10".to_owned(), - surrogates: SortedVec::new(vec![1u32]), - }, - }], - ); - m.insert( - 20u32, - vec![PassiveReadKey { - engine_key: EngineKeySet::Document { - collection: "coll_20".to_owned(), - surrogates: SortedVec::new(vec![2u32]), - }, - }], - ); - m - }, + passive_reads: BTreeMap::from([( + passive_vshard, + vec![PassiveReadKey { + engine_key: EngineKeySet::Document { + collection: "passive_col".to_owned(), + surrogates: SortedVec::new(vec![42u32]), + }, + }], + )]), + expected: BTreeMap::from([(row("passive_col", 42), Some(b"v".to_vec()))]), }; - let tx_class = TxClass::new_dependent( ReadWriteSet::new(vec![]), write_set, vec![], TenantId::new(1), + DatabaseId::DEFAULT, spec, VersionedReadSet::default(), ) - .expect("valid"); - - let mut waiting_for = BTreeSet::new(); - waiting_for.insert(10u32); - waiting_for.insert(20u32); - - let mut barrier = PendingDependentBarrier { - txn: SequencedTxn { - epoch: 1, - position: 0, - tx_class, - epoch_system_ms: 0, - epoch_vshard_txn_count: 0, - lock_owner: None, - }, - lock_owner: TxnId::new(1, 0), - waiting_for, - received: BTreeMap::new(), - timeout_at: Instant::now() + Duration::from_secs(30), - }; - - // Only one passive delivers — barrier should still be incomplete. - barrier.waiting_for.remove(&10u32); - barrier.received.insert( - 10u32, - vec![( - PassiveReadKeyId { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "coll_10"), - surrogate: 1, - }, - Value::Integer(42), - )], - ); + .expect("valid dependent TxClass"); - assert!(!barrier.is_complete(), "still waiting for vshard 20"); + let participants: Vec = tx_class + .participating_vshards() + .iter() + .map(|v| v.as_u32()) + .collect(); + assert!(participants.contains(&passive_vshard)); + let active = tx_class.active_vshards().expect("active vShards"); + assert_eq!(active.len(), 2); + assert!(!active.contains(&passive_vshard)); +} - // Both deliver — now complete. - barrier.waiting_for.remove(&20u32); - barrier.received.insert( - 20u32, - vec![( - PassiveReadKeyId { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "coll_20"), - surrogate: 2, - }, - Value::Integer(99), - )], +/// The barrier completes once every passive vShard's result is in, and the +/// values it injects are the ones the log carried. +#[test] +fn a_barrier_completes_once_every_passive_result_is_in() { + let passive: BTreeSet = [10, 20].into_iter().collect(); + let mut log = BarrierLog::default(); + log.note(read(10, "coll_10", b"a")); + assert_eq!(log.outcome(&passive), BarrierOutcome::Waiting); + log.note(read(20, "coll_20", b"b")); + assert_eq!(log.outcome(&passive), BarrierOutcome::Complete); + let injected = log.injected_reads(); + assert_eq!( + injected.get(&row("coll_10", 1)), + Some(&Value::Bytes(b"a".to_vec())) ); - - assert!(barrier.is_complete()); - let injected = barrier.assemble_injected_reads(); assert_eq!(injected.len(), 2); } +/// A timeout entry that comes before a passive result times the barrier +/// out, whatever arrives after it. #[test] -fn dependent_barrier_timeout_detected() { - let txn = make_dependent_txn(7); - let lock_owner = TxnId::new(txn.epoch, txn.position); - let barrier = PendingDependentBarrier { - txn, - lock_owner, - waiting_for: { - let mut s = BTreeSet::new(); - s.insert(7u32); - s - }, - received: BTreeMap::new(), - // Timeout already in the past. - timeout_at: Instant::now() - Duration::from_millis(1), - }; - - assert!( - barrier.is_timed_out(), - "barrier should report timeout when timeout_at is in the past" - ); +fn a_logged_timeout_before_a_result_times_the_barrier_out() { + let passive: BTreeSet = [10].into_iter().collect(); + let mut log = BarrierLog::default(); + log.note(BarrierEvent::Timeout); + log.note(read(10, "coll_10", b"a")); + assert_eq!(log.outcome(&passive), BarrierOutcome::TimedOut); } -/// Happy-path test: passive delivers, active assembles injected_reads correctly. +/// Values that match the coordinator's read pass. A moved value drifts. #[test] -fn read_result_event_assembles_correctly() { - let event = ReadResultEvent { - epoch: 5, - position: 2, - passive_vshard: 17, - tenant_id: TenantId::new(3), - values: vec![( - PassiveReadKeyId { - collection: QualifiedCollection::new(DatabaseId::DEFAULT, "coll"), - surrogate: 100, - }, - Value::Float(std::f64::consts::PI), - )], +fn read_values_are_checked_against_the_coordinators_read() { + let spec = DependentReadSpec { + passive_reads: BTreeMap::new(), + expected: BTreeMap::from([(row("coll", 1), Some(b"v1".to_vec()))]), }; - - assert_eq!(event.epoch, 5); - assert_eq!(event.passive_vshard, 17); - assert_eq!(event.values.len(), 1); - assert_eq!(event.values[0].0.collection.as_str(), "coll"); + let same = BTreeMap::from([(row("coll", 1), Value::Bytes(b"v1".to_vec()))]); + let moved = BTreeMap::from([(row("coll", 1), Value::Bytes(b"v2".to_vec()))]); + assert_eq!(expected_reads_drift(&spec, &same), None); + assert!(expected_reads_drift(&spec, &moved).is_some()); } From bf7d8cf3207341a5e731ecc54c8b9952fe686420 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sun, 11 Oct 2026 05:03:13 +0800 Subject: [PATCH 17/19] feat(failpoints): scope fail points to a node An in-process cluster shares one fail-point registry across its nodes, so a point could only target a node by encoding the node id in its name. Fail points now carry an explicit scope (any node or one node), evaluated against the scope of the core, producer, or raft group that reaches them. - Split the fail-point module in nodedb-types into action, scope, registry, guard, env, and macro files, with FailGuard::for_node to arm a point on one node. - Read NODEDB_FAILPOINTS entries as `name=action` for every node or `name@node=action` for node N only. - Give each Data Plane core and its event producer the node's scope, and evaluate the Raft commit gate, WAL, funnel, and catalog gates in it. - Re-export the test API from nodedb-test-support behind a failpoints feature, and move the cluster tests onto node-scoped guards. - Add a cluster test that holds one replica's apply while fast-path and Calvin writes hit the same rows, then checks byte-equal convergence. --- CONTRIBUTING.md | 2 +- nodedb-cluster-tests/Cargo.toml | 6 +- .../lagging_catalog.rs | 11 +- .../cases/calvin_backup_cut_order.rs | 2 +- .../calvin_dependent_read_early_result.rs | 2 +- ...alvin_dependent_read_follower_promotion.rs | 9 +- .../cases/calvin_dependent_read_restart.rs | 2 +- .../cases/calvin_dependent_read_timeout.rs | 2 +- .../cases/calvin_duplicate_redo_copy.rs | 2 +- .../cases/calvin_leader_change_mid_txn.rs | 2 +- .../cases/calvin_old_term_redo_copy.rs | 11 +- .../cases/calvin_remote_participant_report.rs | 2 +- .../cases/calvin_replica_content.rs | 37 ++ .../cases/calvin_replica_convergence.rs | 49 +-- .../cases/calvin_replica_held_apply.rs | 317 ++++++++++++++++ .../calvin_superseded_collection_cluster.rs | 2 +- .../cases/cluster_backup_remote_cut.rs | 11 +- .../cases/cluster_pitr_restore_point.rs | 2 +- .../cases/cluster_restore_retry.rs | 2 +- .../cases/committed_publish_failover.rs | 17 +- .../cases/committed_publish_metadata_lag.rs | 9 +- .../cases/graph_wcc_core_error_cross_node.rs | 2 +- .../install_snapshot_multi_core_fault.rs | 13 +- .../tests/common_suite/cases/mod.rs | 1 + .../cases/move_tenant_reissue_retry.rs | 2 +- .../cases/timeseries_ring_drop_catch_up.rs | 7 +- .../cases/trigger_firing_failover.rs | 7 +- nodedb-raft/Cargo.toml | 4 +- nodedb-raft/src/fail_gate.rs | 12 +- nodedb-test-support/Cargo.toml | 5 + nodedb-test-support/src/booted_state.rs | 1 + .../node/lifecycle/spawn_full.rs | 1 + nodedb-test-support/src/core_loop_runner.rs | 5 + nodedb-test-support/src/fail_point.rs | 9 + nodedb-test-support/src/lib.rs | 2 + .../src/pgwire_harness/multicore.rs | 1 + .../src/pgwire_harness/restart.rs | 1 + .../src/pgwire_harness/start.rs | 1 + nodedb-types/src/fail_point.rs | 347 ------------------ nodedb-types/src/fail_point/action.rs | 26 ++ nodedb-types/src/fail_point/env.rs | 181 +++++++++ nodedb-types/src/fail_point/guard.rs | 45 +++ nodedb-types/src/fail_point/macros.rs | 69 ++++ nodedb-types/src/fail_point/mod.rs | 44 +++ nodedb-types/src/fail_point/registry.rs | 224 +++++++++++ nodedb-types/src/fail_point/scope.rs | 27 ++ nodedb-wal/src/uring_writer.rs | 5 +- nodedb-wal/src/writer/durability.rs | 5 +- nodedb/src/bootstrap/data_plane.rs | 2 + .../src/control/backup/cut_capture/collect.rs | 6 +- .../backup/restore/redo_reissue/reissue.rs | 12 +- .../post_apply/async_dispatch/collection.rs | 16 +- .../scheduler/driver/core/passive_read.rs | 4 +- .../scheduler/driver/core/redo_propose.rs | 5 +- .../cluster/metadata_applier/catalog_ddl.rs | 22 +- .../cluster/metadata_applier/dispatch.rs | 7 +- .../control/cluster/metadata_applier/mod.rs | 4 +- .../cluster/snapshot_install/dispatch.rs | 11 +- .../apply_loop/calvin_redo.rs | 7 +- .../apply_loop/redo_chunk.rs | 10 +- nodedb/src/control/fail_gate.rs | 146 ++++---- .../src/control/pitr/restore_point/create.rs | 6 +- .../control/planner/calvin/submit/local.rs | 2 +- .../security/permission_tree/event_handler.rs | 6 +- .../src/control/server/native/session/run.rs | 5 +- .../ddl/neutral/tenant/move_tenant/cutover.rs | 8 +- .../shared/session/commit/single_shard.rs | 16 +- .../src/data/executor/core_loop/accessors.rs | 16 + .../src/data/executor/core_loop/event_emit.rs | 5 + nodedb/src/data/executor/core_loop/open.rs | 3 + nodedb/src/data/executor/core_loop/state.rs | 3 +- nodedb/src/data/executor/core_loop/tick.rs | 10 +- .../handlers/control/calvin/static_stage.rs | 2 +- .../executor/handlers/control/reindex/csr.rs | 3 +- .../executor/handlers/control/reindex/fts.rs | 3 +- .../executor/handlers/control/reindex/hold.rs | 17 +- .../src/data/executor/handlers/graph_wcc.rs | 7 +- nodedb/src/data/executor/wal_replay/kv.rs | 2 +- nodedb/src/data/executor/wal_replay_all.rs | 4 +- nodedb/src/data/runtime/params.rs | 3 + nodedb/src/data/runtime/spawn.rs | 2 + nodedb/src/event/bus.rs | 46 ++- nodedb/src/wal/redo/replay.rs | 4 +- .../cases/event_restart_watermark_catchup.rs | 5 +- 84 files changed, 1370 insertions(+), 606 deletions(-) create mode 100644 nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_held_apply.rs create mode 100644 nodedb-test-support/src/fail_point.rs delete mode 100644 nodedb-types/src/fail_point.rs create mode 100644 nodedb-types/src/fail_point/action.rs create mode 100644 nodedb-types/src/fail_point/env.rs create mode 100644 nodedb-types/src/fail_point/guard.rs create mode 100644 nodedb-types/src/fail_point/macros.rs create mode 100644 nodedb-types/src/fail_point/mod.rs create mode 100644 nodedb-types/src/fail_point/registry.rs create mode 100644 nodedb-types/src/fail_point/scope.rs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 75f4aa99d..652651b20 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -153,7 +153,7 @@ NodeDB routes all DML through `EngineRules` in `nodedb-sql`. This is the single **Integration tests** — place in `nodedb/tests/` or the relevant crate's `tests/` directory. Test public API only. Use `tests/common/mod.rs` for shared helpers (not `tests/common.rs`). -**Failpoint tests** — for fault injection and panic recovery, use the `failpoints` feature flag and `FailGuard` from `nodedb::fail_point`. Compile with `--features failpoints`. +**Failpoint tests** — for fault injection and panic recovery, use the `failpoints` feature flag and `FailGuard` from `nodedb::fail_point`. Compile with `--features failpoints`. An in-process cluster shares one fail-point registry across its nodes. Arm a point on one node with `FailGuard::for_node(node_id, name, action)`, never by putting the node in the name. A spawned server takes `NODEDB_FAILPOINTS='name=action'` for every node, or `name@node=action` for node `N` only. **Cluster tests** — 3-node integration tests live in `nodedb/tests/` and are annotated with the `cluster` nextest group. They serialize automatically via `.config/nextest.toml` — don't run them with `cargo test`. diff --git a/nodedb-cluster-tests/Cargo.toml b/nodedb-cluster-tests/Cargo.toml index f36ab2553..bf5bc827f 100644 --- a/nodedb-cluster-tests/Cargo.toml +++ b/nodedb-cluster-tests/Cargo.toml @@ -12,7 +12,11 @@ path = "src/lib.rs" [features] # Arms the in-process fail points the cluster tests park writes at. -failpoints = ["nodedb/failpoints", "nodedb-types/failpoints"] +failpoints = [ + "nodedb/failpoints", + "nodedb-types/failpoints", + "nodedb-test-support/failpoints", +] [dependencies] diff --git a/nodedb-cluster-tests/tests/common_suite/cases/backup_schedule_failover/lagging_catalog.rs b/nodedb-cluster-tests/tests/common_suite/cases/backup_schedule_failover/lagging_catalog.rs index 6f5a71565..bc69e9048 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/backup_schedule_failover/lagging_catalog.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/backup_schedule_failover/lagging_catalog.rs @@ -4,8 +4,8 @@ //! instead of rerunning the due minute. use nodedb::control::backup::schedule::envelope_name; -use nodedb::control::cluster::metadata_applier::backup_mark_fail_point; -use nodedb_types::fail_point::FailGuard; +use nodedb::control::cluster::metadata_applier::BACKUP_MARK_FAIL_POINT; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use super::fixture::{DATABASE, Fixture}; @@ -26,9 +26,10 @@ async fn a_new_leader_whose_catalog_lags_never_reruns_the_due_minute() { .iter() .filter(|node| node.node_id != leader) .map(|node| { - FailGuard::fail( - &backup_mark_fail_point(node.node_id), - "held back by the test", + FailGuard::for_node( + node.node_id, + BACKUP_MARK_FAIL_POINT, + FailAction::Fail("held back by the test".to_string()), ) }) .collect(); diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_order.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_order.rs index e3af27706..78831172e 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_order.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_backup_cut_order.rs @@ -29,7 +29,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; -use nodedb_types::fail_point::{FailAction, FailGuard}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use super::calvin_multishard_fixture::Fixture; use super::calvin_replica_content::{run_retrying, strict_session, sum_ddl, vshard_of}; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_early_result.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_early_result.rs index ecdb7738e..e34fc8b4e 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_early_result.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_early_result.rs @@ -23,7 +23,7 @@ use std::time::Duration; -use nodedb_types::fail_point::{FailAction, FailGuard}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use super::calvin_dependent_read_fixture::{ ItemMove, assert_no_apply_halt, stored_barrier_rows, waiting_barrier_txns, diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_follower_promotion.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_follower_promotion.rs index 97210f163..d7b886470 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_follower_promotion.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_follower_promotion.rs @@ -24,8 +24,8 @@ use std::time::Duration; -use nodedb::control::cluster::metadata_applier::metadata_apply_hold_point; -use nodedb_types::fail_point::{FailAction, FailGuard}; +use nodedb::control::cluster::metadata_applier::METADATA_APPLY_HOLD_POINT; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use super::calvin_dependent_read_fixture::{ItemMove, assert_no_apply_halt, waiting_barrier_txns}; use super::calvin_replica_content::strict_session; @@ -60,8 +60,9 @@ async fn a_promoted_follower_commits_the_move_without_halting() { let meta_dir = tempfile::tempdir().expect("metadata gate directory"); let meta_release = meta_dir.path().join("release"); let meta_parked = meta_dir.path().join("release.parked"); - let _meta_hold = FailGuard::install( - &metadata_apply_hold_point(follower_id), + let _meta_hold = FailGuard::for_node( + follower_id, + METADATA_APPLY_HOLD_POINT, FailAction::WaitForFile(meta_release.clone()), ); nodes[coordinator] diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_restart.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_restart.rs index a6f5f6e37..87b832840 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_restart.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_restart.rs @@ -25,7 +25,7 @@ use std::time::Duration; -use nodedb_types::fail_point::{FailAction, FailGuard}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use super::calvin_dependent_read_fixture::{ ItemMove, assert_no_apply_halt, read_entries, stored_barrier_rows, waiting_barrier_txns, diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_timeout.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_timeout.rs index b997a493a..c8eb74ecf 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_timeout.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_dependent_read_timeout.rs @@ -21,7 +21,7 @@ use std::time::Duration; -use nodedb_types::fail_point::{FailAction, FailGuard}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use super::calvin_dependent_read_fixture::{ ItemMove, assert_no_apply_halt, reads_applied, stored_barrier_rows, waiting_barrier_txns, diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_duplicate_redo_copy.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_duplicate_redo_copy.rs index 50e53e18f..a384be6a2 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_duplicate_redo_copy.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_duplicate_redo_copy.rs @@ -24,7 +24,7 @@ use std::sync::atomic::Ordering; use std::time::Duration; -use nodedb_types::fail_point::{FailAction, FailGuard}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use super::calvin_multishard_fixture::{Fixture, tags}; use super::calvin_replica_content::{ diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_leader_change_mid_txn.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_leader_change_mid_txn.rs index b39473740..a96679bd9 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_leader_change_mid_txn.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_leader_change_mid_txn.rs @@ -24,7 +24,7 @@ use std::time::Duration; -use nodedb_types::fail_point::{FailAction, FailGuard}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use super::calvin_multishard_fixture::{Fixture, keyed_ddl}; use super::vshard_names::distinct_vshard_collections; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_old_term_redo_copy.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_old_term_redo_copy.rs index 0b0d8fe7a..ac5efb88c 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_old_term_redo_copy.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_old_term_redo_copy.rs @@ -11,8 +11,8 @@ //! 2. The fail gate `calvin::before_redo_propose::` holds the //! entries slice after its vote and resolve, on the leader `L` of the //! entries group `G`. The balance slice installs meanwhile. -//! 3. The test arms `raft::hold_commit::node::group`, then releases -//! the redo. `L` proposes it, the entry replicates to every replica, and +//! 3. The test arms `raft::hold_commit::group` on node `L` only, then +//! releases the redo. `L` proposes it, the entry replicates to every replica, and //! `L` commits nothing. //! 4. The test moves `G`'s leadership to another replica `N`. The entry of //! the old term commits with `N`'s first entry of the new term. `N` @@ -30,7 +30,7 @@ use std::sync::atomic::Ordering; use std::time::Duration; -use nodedb_types::fail_point::{FailAction, FailGuard}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use super::calvin_multishard_fixture::{Fixture, tags}; use super::calvin_replica_content::{ @@ -170,8 +170,9 @@ async fn an_old_term_redo_copy_committed_by_the_new_leader_installs_once() { let held_status = group_status(&fx.cluster.nodes[old], group) .expect("the old leader hosts the entries group"); - let _commit_gate = FailGuard::install( - &format!("raft::hold_commit::node{old_leader}::group{group}"), + let _commit_gate = FailGuard::for_node( + old_leader, + &format!("raft::hold_commit::group{group}"), FailAction::WaitForFile(commit_release.clone()), ); std::fs::write(&propose_release, b"release").expect("release the held redo"); diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_remote_participant_report.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_remote_participant_report.rs index 929a7bafe..b252c95f9 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_remote_participant_report.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_remote_participant_report.rs @@ -33,8 +33,8 @@ use crate::common; use common::cluster_harness::shared_steps::{fail_stopped, sequencer_admitted}; use common::cluster_harness::{TestCluster, TestClusterNode, wait_for}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use nodedb_test_support::native_harness::send_sql; -use nodedb_types::fail_point::{FailAction, FailGuard}; use nodedb_types::{DatabaseId, TenantId}; use super::ts_native_ingest::{ diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_content.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_content.rs index c6eb12ead..2fc5faec6 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_content.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_content.rs @@ -144,6 +144,43 @@ pub(super) async fn first_column(client: &tokio_postgres::Client, sql: &str) -> .collect() } +/// A served number column as `f64`. +fn number(text: &str, what: &str) -> f64 { + text.parse() + .unwrap_or_else(|e| panic!("{what} `{text}` is not a number: {e}")) +} + +/// Each of the `accounts` rows `acc-0` to `acc-{accounts - 1}` of `accts` +/// serves a balance equal to the sum of its served `entries` amounts. +pub(super) async fn assert_balances_match_entries( + client: &tokio_postgres::Client, + accts: &str, + entries: &str, + accounts: usize, +) { + for account in 0..accounts { + let balances = first_column( + client, + &format!("SELECT balance FROM {accts} WHERE id = 'acc-{account}'"), + ) + .await; + let [balance] = balances.as_slice() else { + panic!("acc-{account} has one row, found balances {balances:?}"); + }; + let amounts = first_column( + client, + &format!("SELECT amount FROM {entries} WHERE account_id = 'acc-{account}'"), + ) + .await; + let expected: f64 = amounts.iter().map(|a| number(a, "amount")).sum(); + assert_eq!( + number(balance, "balance"), + expected, + "acc-{account}'s balance is the sum of its entries {amounts:?}" + ); + } +} + /// The content of each vShard `collections` home to, as `node` stores it. pub(super) async fn local_content( node: &TestClusterNode, diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_convergence.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_convergence.rs index e2978d72f..78d9e5d09 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_convergence.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_convergence.rs @@ -23,8 +23,8 @@ use super::calvin_multishard_fixture::{Fixture, keyed_ddl}; use super::calvin_replica_content::{ - first_column, names_on_distinct_vshards, run_retrying, strict_session, sum_ddl, vshard_of, - wait_replicas_agree, + assert_balances_match_entries, names_on_distinct_vshards, run_retrying, strict_session, + sum_ddl, vshard_of, wait_replicas_agree, }; /// Rounds each session runs. @@ -133,43 +133,6 @@ async fn truncating_writes(client: &tokio_postgres::Client, names: &Names) { } } -/// A served number column as `f64`. -fn number(text: &str, what: &str) -> f64 { - text.parse() - .unwrap_or_else(|e| panic!("{what} `{text}` is not a number: {e}")) -} - -/// Each account's served balance equals the sum of its served entries. -async fn assert_balances_match_entries(client: &tokio_postgres::Client, names: &Names) { - for account in 0..ACCOUNTS { - let balances = first_column( - client, - &format!( - "SELECT balance FROM {} WHERE id = 'acc-{account}'", - names.accts - ), - ) - .await; - let [balance] = balances.as_slice() else { - panic!("acc-{account} has one row, found balances {balances:?}"); - }; - let amounts = first_column( - client, - &format!( - "SELECT amount FROM {} WHERE account_id = 'acc-{account}'", - names.entries - ), - ) - .await; - let expected: f64 = amounts.iter().map(|a| number(a, "amount")).sum(); - assert_eq!( - number(balance, "balance"), - expected, - "acc-{account}'s balance is the sum of its entries {amounts:?}" - ); - } -} - #[tokio::test(flavor = "multi_thread", worker_threads = 6)] async fn replicas_converge_under_concurrent_fast_path_and_calvin_writes() { let [accts, entries, notes, scratch] = @@ -241,7 +204,13 @@ async fn replicas_converge_under_concurrent_fast_path_and_calvin_writes() { "{collection} holds rows after the workload; equal empty replicas prove nothing" ); } - assert_balances_match_entries(&fx.coordinator().client, &names).await; + assert_balances_match_entries( + &fx.coordinator().client, + &names.accts, + &names.entries, + ACCOUNTS, + ) + .await; fx.cluster.shutdown().await; } diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_held_apply.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_held_apply.rs new file mode 100644 index 000000000..5dc0bc526 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_replica_held_apply.rs @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A replica whose apply is held while fast-path writes and Calvin +//! transactions hit the same rows converges byte-equal once released, and +//! the hold touches that replica only. +//! +//! A 3-node cluster holds three collections, each on its own vShard: +//! - `accts`: strict, its `balance` is the materialized sum of +//! `entries.amount` per account, +//! - `entries`: strict, the sum's source, +//! - `notes`: keyed. +//! +//! The fail gate `funnel::before_dispatch::` is armed for one +//! follower `F` of the `notes` group, through a node-scoped guard. Every +//! node shares the one fail-point registry, so the scope alone keeps the +//! other nodes running. +//! +//! - A probe insert into `notes` commits. Every other replica of the group +//! installs it, and `F` still stores the `notes` content it had before. +//! - Two sessions on the other two nodes run at once: single-shard +//! autocommit writes to `accts` and `notes`, and Calvin transactions that +//! insert `entries` rows and update the same `notes` rows. Both finish +//! while `F` is held, and `F` still stores its old `notes` content. +//! - The gate releases. Each vShard's documents and index entries become +//! byte-equal on every replica, read from each node's own cores. Each +//! account's served balance equals the sum of its served entries, and no +//! node halted its Calvin apply. +//! +//! Requires `--features failpoints`. + +#![cfg(feature = "failpoints")] + +use std::path::Path; +use std::time::Duration; + +use nodedb_test_support::fail_point::{FailAction, FailGuard}; + +use super::calvin_dependent_read_fixture::assert_no_apply_halt; +use super::calvin_multishard_fixture::{Fixture, keyed_ddl}; +use super::calvin_replica_content::{ + VShardContent, assert_balances_match_entries, local_content, names_on_distinct_vshards, + run_retrying, strict_session, sum_ddl, vshard_of, wait_replicas_agree, +}; +use crate::common::cluster_harness::shared_steps::group_members; +use crate::common::cluster_harness::{GroupLeader, TestCluster, TestClusterNode, wait_for_async}; + +/// Rounds each session runs. +const ROUNDS: usize = 10; + +/// Accounts in `accts`, `acc-0` to `acc-{ACCOUNTS - 1}`. +const ACCOUNTS: usize = 3; + +/// Seeded `notes` rows, `n0` to `n{NOTES - 1}`. +const NOTES: usize = 4; + +/// How long each step around the hold gets. +const STEP_DEADLINE: Duration = Duration::from_secs(30); + +/// Pause between two polls of a step. +const STEP_POLL: Duration = Duration::from_millis(50); + +/// How long the workload gets to finish while `F` is held. +const HOLD_WINDOW: Duration = Duration::from_secs(90); + +/// The three collections of the case. +struct Names { + accts: String, + entries: String, + notes: String, +} + +impl Names { + fn all(&self) -> [&str; 3] { + [ + self.accts.as_str(), + self.entries.as_str(), + self.notes.as_str(), + ] + } +} + +/// Single-shard autocommit writes: an `accts` column the sum leaves alone, +/// and a `notes` row the Calvin session also updates. +async fn fast_path_writes(client: &tokio_postgres::Client, names: &Names) { + for k in 0..ROUNDS { + let account = k % ACCOUNTS; + let note = k % NOTES; + let statements = [ + format!( + "UPDATE {} SET owner = 'fp{k}' WHERE id = 'acc-{account}'", + names.accts + ), + format!( + "UPDATE {} SET v = 'fp{k}' WHERE id = 'n{note}'", + names.notes + ), + ]; + for sql in &statements { + run_retrying(client, &format!("fast path round {k}"), sql).await; + } + } +} + +/// Cross-shard transactions: each inserts an entry, which moves its +/// account's balance on another vShard, and updates a `notes` row. +async fn calvin_writes(client: &tokio_postgres::Client, names: &Names) { + for k in 0..ROUNDS { + let account = k % ACCOUNTS; + let note = k % NOTES; + let amount = k + 1; + let txn = format!( + "BEGIN; \ + INSERT INTO {entries} (id, account_id, amount) \ + VALUES ('e{k}', 'acc-{account}', '{amount}'); \ + UPDATE {notes} SET v = 'cv{k}' WHERE id = 'n{note}'; \ + COMMIT", + entries = names.entries, + notes = names.notes, + ); + run_retrying(client, &format!("calvin round {k}"), &txn).await; + } +} + +/// `node`'s stored content of the vShard `collection` homes to. +async fn vshard_content(node: &TestClusterNode, collection: &str) -> VShardContent { + local_content(node, &[collection]) + .await + .remove(&vshard_of(collection)) + .unwrap_or_default() +} + +/// The index in `cluster.nodes` of node `node_id`. +fn index_of(cluster: &TestCluster, node_id: u64) -> usize { + cluster + .nodes + .iter() + .position(|node| node.node_id == node_id) + .unwrap_or_else(|| panic!("node {node_id} is live")) +} + +/// Wait until the held apply parked at the gate, so `parked` exists, and +/// every node of `replicas` stores `collection` content other than `before`. +async fn wait_others_moved_on( + cluster: &TestCluster, + replicas: &[usize], + collection: &str, + before: &VShardContent, + parked: &Path, +) { + wait_for_async( + "the held follower parks, and every other replica installs the probe", + STEP_DEADLINE, + STEP_POLL, + || async move { + if !parked.exists() { + return false; + } + for idx in replicas { + if vshard_content(&cluster.nodes[*idx], collection).await == *before { + return false; + } + } + true + }, + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn a_replica_held_under_fast_path_and_calvin_writes_converges_once_released() { + let [accts, entries, notes] = + names_on_distinct_vshards(["held_accts", "held_entries", "held_notes"]); + let names = Names { + accts, + entries, + notes, + }; + let [accts_ddl, entries_ddl, sum] = sum_ddl(&names.entries, &names.accts); + let fx = Fixture::spawn(&[accts_ddl, entries_ddl, keyed_ddl(&names.notes)]).await; + fx.cluster + .exec_ddl_on_any_leader(&sum) + .await + .unwrap_or_else(|e| panic!("{sum}: {e}")); + for collection in names.all() { + fx.wait_group_mounted(collection).await; + } + + let seeder = &fx.cluster.nodes[0].client; + for account in 0..ACCOUNTS { + run_retrying( + seeder, + "seed an account", + &format!( + "INSERT INTO {} (id, owner, balance) VALUES ('acc-{account}', 'seed', '0')", + names.accts + ), + ) + .await; + } + for note in 0..NOTES { + run_retrying( + seeder, + "seed a note", + &format!( + "INSERT INTO {} (id, v) VALUES ('n{note}', 'seed')", + names.notes + ), + ) + .await; + } + fx.converge().await; + // Stable leaders: a leadership move onto the held follower would make + // the workload wait for the hold. + fx.cluster.wait_for_preferred_leaders(STEP_DEADLINE).await; + + let group = fx.cluster.nodes[0] + .group_id_for_collection(&names.notes) + .expect("the notes data group"); + let leader = fx + .cluster + .wait_for_group_leader(group, GroupLeader::Any, STEP_DEADLINE) + .await; + let held_id = group_members(&fx.cluster.nodes[0], group) + .into_iter() + .find(|member| *member != leader) + .expect("the notes group has a follower"); + let held = index_of(&fx.cluster, held_id); + let others: Vec = (0..fx.cluster.nodes.len()) + .filter(|idx| *idx != held) + .collect(); + let &[fast_node, calvin_node] = others.as_slice() else { + panic!("a 3-node cluster has two nodes besides the held one, found {others:?}"); + }; + let replicas: Vec = others + .iter() + .copied() + .filter(|idx| fx.cluster.nodes[*idx].replicates_data_group(group)) + .collect(); + assert!( + !replicas.is_empty(), + "the notes group has a replica besides the held follower" + ); + let before = vshard_content(&fx.cluster.nodes[held], &names.notes).await; + + let gate_dir = tempfile::tempdir().expect("gate directory"); + let release = gate_dir.path().join("release"); + let parked = gate_dir.path().join("release.parked"); + let gate = FailGuard::for_node( + held_id, + &format!("funnel::before_dispatch::{}", names.notes), + FailAction::WaitForFile(release.clone()), + ); + + let fast = &fx.cluster.nodes[fast_node].client; + run_retrying( + fast, + "probe insert", + &format!("INSERT INTO {} (id, v) VALUES ('probe', 'p')", names.notes), + ) + .await; + wait_others_moved_on(&fx.cluster, &replicas, &names.notes, &before, &parked).await; + let held_after_probe = vshard_content(&fx.cluster.nodes[held], &names.notes).await; + + let calvin = strict_session(&fx.cluster.nodes[calvin_node]).await; + let mut workload = Box::pin(async { + tokio::join!( + fast_path_writes(fast, &names), + calvin_writes(&calvin, &names), + ) + }); + let finished_while_held = tokio::time::timeout(HOLD_WINDOW, workload.as_mut()) + .await + .is_ok(); + let held_after_workload = vshard_content(&fx.cluster.nodes[held], &names.notes).await; + + // Release before any assertion, so a failed one leaves no apply parked. + std::fs::write(&release, b"release").expect("release the held apply"); + drop(gate); + if !finished_while_held { + workload.as_mut().await; + } + drop(workload); + assert!( + held_after_probe == before && held_after_workload == before, + "node {held_id} installed a notes write while its apply was held: {} entries before, \ + {} after the probe, {} after the workload", + before.len(), + held_after_probe.len(), + held_after_workload.len() + ); + assert!( + finished_while_held, + "the workload did not finish within {HOLD_WINDOW:?} while only node {held_id}'s \ + notes apply was held" + ); + + fx.converge().await; + let content = wait_replicas_agree(&fx.cluster, &names.all()).await; + for collection in names.all() { + assert!( + content + .get(&vshard_of(collection)) + .is_some_and(|stored| !stored.is_empty()), + "{collection} holds rows after the workload; equal empty replicas prove nothing" + ); + } + assert_balances_match_entries( + &fx.coordinator().client, + &names.accts, + &names.entries, + ACCOUNTS, + ) + .await; + assert_no_apply_halt(&fx.cluster); + + fx.cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/calvin_superseded_collection_cluster.rs b/nodedb-cluster-tests/tests/common_suite/cases/calvin_superseded_collection_cluster.rs index f2e21a287..14e4c54ee 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/calvin_superseded_collection_cluster.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/calvin_superseded_collection_cluster.rs @@ -24,7 +24,7 @@ use std::time::Duration; -use nodedb_types::fail_point::{FailAction, FailGuard}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use tokio_postgres::error::SqlState; use super::calvin_multishard_fixture::{Fixture, keyed_ddl}; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/cluster_backup_remote_cut.rs b/nodedb-cluster-tests/tests/common_suite/cases/cluster_backup_remote_cut.rs index 3ce94ab65..e9d9f39ac 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/cluster_backup_remote_cut.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/cluster_backup_remote_cut.rs @@ -4,8 +4,8 @@ //! //! The backup's coordinator snapshots every vShard from the leader of its //! group. The test parks a write on the collection's group leader only, at -//! the fail gate `funnel::before_dispatch::node::`, between -//! its record and its core. The coordinator is another node, so its own +//! the fail gate `funnel::before_dispatch::` armed for that node, +//! between its record and its core. The coordinator is another node, so its own //! replica applies the write and its own cut passes. The leader must take the //! same cut before it snapshots: the backup waits until the parked write //! applies there, and the envelope holds the row. After the purge settles on @@ -19,10 +19,10 @@ use std::time::Duration; use bytes::Bytes; use futures::{SinkExt, StreamExt}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use nodedb_types::backup_envelope::{ DEFAULT_MAX_TOTAL_BYTES, DatabaseDataSection, parse_encrypted, }; -use nodedb_types::fail_point::{FailAction, FailGuard}; use crate::common; use common::cluster_harness::wait::wait_for; @@ -192,8 +192,9 @@ async fn a_backup_waits_for_a_write_held_on_a_remote_source_node() { // Park the write on the source node only. let gate_dir = tempfile::tempdir().expect("gate tempdir"); let release = gate_dir.path().join("release-source-apply"); - let _gate = FailGuard::install( - &format!("funnel::before_dispatch::node{source}::{COLLECTION}"), + let _gate = FailGuard::for_node( + source, + &format!("funnel::before_dispatch::{COLLECTION}"), FailAction::WaitForFile(release.clone()), ); diff --git a/nodedb-cluster-tests/tests/common_suite/cases/cluster_pitr_restore_point.rs b/nodedb-cluster-tests/tests/common_suite/cases/cluster_pitr_restore_point.rs index 7ab977aca..5a9f7fbe5 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/cluster_pitr_restore_point.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/cluster_pitr_restore_point.rs @@ -215,7 +215,7 @@ async fn a_cluster_restores_every_group_to_a_restore_point() { #[cfg(feature = "failpoints")] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_ddl_after_the_watermark_is_absent_though_it_precedes_the_point() { - use nodedb_types::fail_point::{FailAction, FailGuard}; + use nodedb_test_support::fail_point::{FailAction, FailGuard}; let root = tempfile::tempdir().expect("storage root"); let pitr = PitrStorage::create(root.path()).expect("PITR storage"); diff --git a/nodedb-cluster-tests/tests/common_suite/cases/cluster_restore_retry.rs b/nodedb-cluster-tests/tests/common_suite/cases/cluster_restore_retry.rs index 93b1612cc..35623482b 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/cluster_restore_retry.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/cluster_restore_retry.rs @@ -16,7 +16,7 @@ use std::collections::HashSet; use std::time::Duration; -use nodedb_types::fail_point::FailGuard; +use nodedb_test_support::fail_point::FailGuard; use crate::common::cluster_harness::shared_steps::{db_detail, drain_backup, try_push_restore}; use crate::common::cluster_harness::{TestCluster, wait_for, wait_for_async}; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/committed_publish_failover.rs b/nodedb-cluster-tests/tests/common_suite/cases/committed_publish_failover.rs index f4737d4c1..15127427e 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/committed_publish_failover.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/committed_publish_failover.rs @@ -27,8 +27,8 @@ use std::path::Path; use std::time::Duration; use nodedb::event::cdc::CdcOffset; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use nodedb_types::DatabaseId; -use nodedb_types::fail_point::{FailAction, FailGuard}; const TOPIC: &str = "committed_failover_feed"; const SRC: &str = "committed_failover_src"; @@ -184,8 +184,9 @@ async fn a_publish_committed_before_a_leader_kill_is_delivered_once() { // Never created: the leader stays parked until it dies. let release = gate_dir.path().join("release"); let parked = gate_dir.path().join("release.parked"); - let _gate = FailGuard::install( - &format!("publish::before_delivery::node{leader_id}"), + let _gate = FailGuard::for_node( + leader_id, + "publish::before_delivery", FailAction::WaitForFile(release), ); wait_parked(&parked, "the leader's delivery parks").await; @@ -223,8 +224,9 @@ async fn a_publish_sent_before_a_leader_kill_is_not_appended_twice() { let hold_dir = tempfile::tempdir().expect("gate directory"); let hold_release = hold_dir.path().join("release"); let hold_parked = hold_dir.path().join("release.parked"); - let _hold = FailGuard::install( - &format!("publish::before_delivery::node{leader_id}"), + let _hold = FailGuard::for_node( + leader_id, + "publish::before_delivery", FailAction::WaitForFile(hold_release.clone()), ); wait_parked(&hold_parked, "the leader's delivery parks").await; @@ -241,8 +243,9 @@ async fn a_publish_sent_before_a_leader_kill_is_not_appended_twice() { // file of the second gate is never created: the leader dies parked. let commit_dir = tempfile::tempdir().expect("gate directory"); let commit_parked = commit_dir.path().join("release.parked"); - let _commit = FailGuard::install( - &format!("publish::before_cursor_commit::node{leader_id}"), + let _commit = FailGuard::for_node( + leader_id, + "publish::before_cursor_commit", FailAction::WaitForFile(commit_dir.path().join("release")), ); std::fs::write(&hold_release, b"release").expect("release the delivery"); diff --git a/nodedb-cluster-tests/tests/common_suite/cases/committed_publish_metadata_lag.rs b/nodedb-cluster-tests/tests/common_suite/cases/committed_publish_metadata_lag.rs index cefee65c5..5fb632be1 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/committed_publish_metadata_lag.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/committed_publish_metadata_lag.rs @@ -20,10 +20,10 @@ use common::cluster_harness::{TestCluster, TestClusterNode, wait_for}; use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; -use nodedb::control::cluster::metadata_applier::metadata_apply_hold_point; +use nodedb::control::cluster::metadata_applier::METADATA_APPLY_HOLD_POINT; use nodedb::event::cdc::CdcOffset; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use nodedb_types::DatabaseId; -use nodedb_types::fail_point::{FailAction, FailGuard}; const TOPIC: &str = "metadata_lag_feed"; const SRC: &str = "metadata_lag_src"; @@ -141,8 +141,9 @@ async fn a_lagging_lease_holder_waits_for_the_topic_and_delivers_once() { let gate_dir = tempfile::tempdir().expect("gate directory"); let release = gate_dir.path().join("release"); let parked = gate_dir.path().join("release.parked"); - let _hold = FailGuard::install( - &metadata_apply_hold_point(b_id), + let _hold = FailGuard::for_node( + b_id, + METADATA_APPLY_HOLD_POINT, FailAction::WaitForFile(release.clone()), ); exec_on(&cluster.nodes[a], &format!("CREATE TOPIC {TOPIC}")).await; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/graph_wcc_core_error_cross_node.rs b/nodedb-cluster-tests/tests/common_suite/cases/graph_wcc_core_error_cross_node.rs index eb64b42a3..c48325062 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/graph_wcc_core_error_cross_node.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/graph_wcc_core_error_cross_node.rs @@ -15,7 +15,7 @@ use std::collections::HashSet; use std::time::Duration; -use nodedb_types::fail_point::FailGuard; +use nodedb_test_support::fail_point::FailGuard; use crate::common::cluster_harness::{TestCluster, wait_for}; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/install_snapshot_multi_core_fault.rs b/nodedb-cluster-tests/tests/common_suite/cases/install_snapshot_multi_core_fault.rs index 0f73d47cc..22bf34134 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/install_snapshot_multi_core_fault.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/install_snapshot_multi_core_fault.rs @@ -4,8 +4,8 @@ //! and a re-install of the same bytes converges to every row on its owning //! core. //! -//! The fail point `snapshot_install::node::core1` makes core 1 of node N -//! report its share as failed after it installed it: the other core holds the +//! The fail point `snapshot_install::core1`, armed for node N, makes core 1 +//! of node N report its share as failed after it installed it: the other core holds the //! snapshot, this one does not count as settled. The re-install clears every //! core before it installs, so no row lands twice or off its core. //! @@ -21,7 +21,7 @@ use nodedb::control::cluster::snapshot_builder::DataPlaneSnapshotBuilder; use nodedb::control::cluster::snapshot_install::SnapshotInstallError; use nodedb::types::TenantId; use nodedb_cluster::SnapshotBuilder; -use nodedb_types::fail_point::FailGuard; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use crate::common::cluster_harness::shared_steps::key_collection; use crate::common::cluster_harness::{TestCluster, wait_for}; @@ -97,9 +97,10 @@ async fn failed_core_install_is_retryable_and_reinstall_converges() { let applier = DataPlaneSnapshotApplier::new(target.shared.clone()); { - let _fault = FailGuard::fail( - &format!("snapshot_install::node{}::core1", target.node_id), - "injected core install fault", + let _fault = FailGuard::for_node( + target.node_id, + "snapshot_install::core1", + FailAction::Fail("injected core install fault".to_string()), ); let err = applier .install(gid, &bytes) diff --git a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs index a246c9eda..1e7a13aa4 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs @@ -39,6 +39,7 @@ mod calvin_read_version_leader_move; mod calvin_remote_participant_report; mod calvin_replica_content; mod calvin_replica_convergence; +mod calvin_replica_held_apply; mod calvin_split_leader; mod calvin_submit_routed_cross_node; mod calvin_superseded_collection_cluster; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/move_tenant_reissue_retry.rs b/nodedb-cluster-tests/tests/common_suite/cases/move_tenant_reissue_retry.rs index c21a882c7..98a9b4e40 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/move_tenant_reissue_retry.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/move_tenant_reissue_retry.rs @@ -15,7 +15,7 @@ use std::time::Duration; -use nodedb_types::fail_point::FailGuard; +use nodedb_test_support::fail_point::FailGuard; use crate::common; use common::cluster_harness::TestCluster; diff --git a/nodedb-cluster-tests/tests/common_suite/cases/timeseries_ring_drop_catch_up.rs b/nodedb-cluster-tests/tests/common_suite/cases/timeseries_ring_drop_catch_up.rs index c71430df4..0942bec47 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/timeseries_ring_drop_catch_up.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/timeseries_ring_drop_catch_up.rs @@ -41,8 +41,8 @@ use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; use nodedb::event::cdc::consume::{ConsumeError, ConsumeParams, consume_local}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; use nodedb_types::DatabaseId; -use nodedb_types::fail_point::{FailAction, FailGuard}; use super::ts_native_ingest::{ ingest_native, ingest_until_accepted, native_session, rejection_warnings, @@ -75,8 +75,9 @@ impl ApplyGate { fn install(directory: &Path, node_id: u64) -> Self { let release = directory.join(format!("release-node{node_id}")); let arrival = directory.join(format!("release-node{node_id}.parked")); - let guard = FailGuard::install( - &format!("funnel::before_dispatch::node{node_id}::{COLLECTION}"), + let guard = FailGuard::for_node( + node_id, + &format!("funnel::before_dispatch::{COLLECTION}"), FailAction::WaitForFile(release.clone()), ); Self { diff --git a/nodedb-cluster-tests/tests/common_suite/cases/trigger_firing_failover.rs b/nodedb-cluster-tests/tests/common_suite/cases/trigger_firing_failover.rs index ca37ff11e..9b51574d3 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/trigger_firing_failover.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/trigger_firing_failover.rs @@ -31,7 +31,7 @@ use std::path::{Path, PathBuf}; use std::time::Duration; use nodedb::event::cdc::CdcOffset; -use nodedb_types::fail_point::{FailAction, FailGuard}; +use nodedb_test_support::fail_point::{FailAction, FailGuard}; const SRC: &str = "firing_failover_src"; const ROWS: [&str; 3] = ["r1", "r2", "r3"]; @@ -81,8 +81,9 @@ impl Gate { fn install(dir: &Path, point: &str, node_id: u64) -> Self { let release = dir.join(format!("{point}-node{node_id}")); let parked = dir.join(format!("{point}-node{node_id}.parked")); - let guard = FailGuard::install( - &format!("trigger::{point}::node{node_id}"), + let guard = FailGuard::for_node( + node_id, + &format!("trigger::{point}"), FailAction::WaitForFile(release.clone()), ); Self { diff --git a/nodedb-raft/Cargo.toml b/nodedb-raft/Cargo.toml index 623b44fc9..50ac35058 100644 --- a/nodedb-raft/Cargo.toml +++ b/nodedb-raft/Cargo.toml @@ -14,8 +14,8 @@ categories = ["algorithms", "network-programming"] [features] default = [] -# Arms the commit gate (`raft::hold_commit::node::group`) cluster tests -# use to keep an entry uncommitted across a leadership change. +# Arms the node-scoped commit gate (`raft::hold_commit::group`) cluster +# tests use to keep an entry uncommitted across a leadership change. failpoints = ["dep:nodedb-types", "nodedb-types/failpoints"] [dependencies] diff --git a/nodedb-raft/src/fail_gate.rs b/nodedb-raft/src/fail_gate.rs index 69809d65d..a79be1ce9 100644 --- a/nodedb-raft/src/fail_gate.rs +++ b/nodedb-raft/src/fail_gate.rs @@ -6,19 +6,19 @@ //! waiting: Raft steps are synchronous, so a gate holds a step back and the //! driver asks again on its next step. -use nodedb_types::fail_point::{FailAction, lookup}; +use nodedb_types::fail_point::{FailAction, FailScope, lookup}; /// Whether the commit gate of `group_id` on `node_id` holds now. Named -/// `raft::hold_commit::node::group`, armed with `wait_file()`. -/// It holds while the file is absent. +/// `raft::hold_commit::group` and scoped to the node, armed with +/// `wait_file()`. It holds while the file is absent. /// /// A leader under the gate counts no quorum, so it commits no new entry. /// The entries it appends still replicate. A test keeps an entry /// uncommitted this way, then moves the leadership: the entry commits only /// in the next term. On its first hold the gate creates `.parked`. pub(crate) fn holds_commit(node_id: u64, group_id: u64) -> bool { - let name = format!("raft::hold_commit::node{node_id}::group{group_id}"); - let Some(FailAction::WaitForFile(path)) = lookup(&name) else { + let name = format!("raft::hold_commit::group{group_id}"); + let Some(FailAction::WaitForFile(path)) = lookup(FailScope::Node(node_id), &name) else { return false; }; if path.exists() { @@ -30,7 +30,7 @@ pub(crate) fn holds_commit(node_id: u64, group_id: u64) -> bool { if !parked.exists() && let Err(e) = std::fs::write(&parked, &name) { - tracing::warn!(gate = %name, error = %e, "fail gate parked marker not created"); + tracing::warn!(gate = %name, node_id, error = %e, "fail gate parked marker not created"); } true } diff --git a/nodedb-test-support/Cargo.toml b/nodedb-test-support/Cargo.toml index c93ed50f0..385462da9 100644 --- a/nodedb-test-support/Cargo.toml +++ b/nodedb-test-support/Cargo.toml @@ -10,6 +10,11 @@ description = "Shared test harness for NodeDB integration tests (not for end-use [lib] path = "src/lib.rs" +[features] +# Re-exports the fail-point test API (`fail_point::FailGuard`) and arms the +# fail points of the nodes this harness spawns. +failpoints = ["nodedb/failpoints", "nodedb-types/failpoints"] + [dependencies] # Production crates this harness drives. nodedb = { workspace = true } diff --git a/nodedb-test-support/src/booted_state.rs b/nodedb-test-support/src/booted_state.rs index 68984ba5d..6cbb67fa8 100644 --- a/nodedb-test-support/src/booted_state.rs +++ b/nodedb-test-support/src/booted_state.rs @@ -186,6 +186,7 @@ async fn boot_node(options: BootOptions) -> Result { let (core_stop_tx, core_stop_rx) = std::sync::mpsc::channel::<()>(); let core = crate::core_loop_runner::spawn_core_loop(crate::core_loop_runner::CoreLoopSpawn { idx: 0, + node_id: state.node_id, num_cores: 1, data_side, core_dir: dir.path().to_path_buf(), diff --git a/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs b/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs index 566288684..004496689 100644 --- a/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs +++ b/nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs @@ -245,6 +245,7 @@ impl TestClusterNode { let core_handle = crate::core_loop_runner::spawn_core_loop(crate::core_loop_runner::CoreLoopSpawn { idx, + node_id, num_cores, data_side, core_dir: data_dir_path.clone(), diff --git a/nodedb-test-support/src/core_loop_runner.rs b/nodedb-test-support/src/core_loop_runner.rs index ef19a6922..1d20df3a5 100644 --- a/nodedb-test-support/src/core_loop_runner.rs +++ b/nodedb-test-support/src/core_loop_runner.rs @@ -63,6 +63,9 @@ pub struct WalReplay { pub struct CoreLoopSpawn { /// Core index within the data plane (0-based). pub idx: usize, + /// The node this core runs on, the scope of its fail points. An + /// in-process cluster runs several nodes, so a test arms one node's core. + pub node_id: u64, /// Total number of Data-Plane cores in this node. The committed-redo apply /// routes each record to `vshard_id % num_cores`, the same rule the /// dispatcher routes requests by. @@ -131,6 +134,7 @@ pub fn event_interest_for(shared: &SharedState) -> Arc { pub fn spawn_core_loop(spawn: CoreLoopSpawn) -> tokio::task::JoinHandle<()> { let CoreLoopSpawn { idx, + node_id, num_cores, data_side, core_dir, @@ -162,6 +166,7 @@ pub fn spawn_core_loop(spawn: CoreLoopSpawn) -> tokio::task::JoinHandle<()> { core_array_catalog, ) .expect("CoreLoop::open_with_array_catalog"); + core.set_fail_scope(node_id); core.set_event_producer(event_producer); core.set_event_interest(event_interest); core.set_num_cores(num_cores); diff --git a/nodedb-test-support/src/fail_point.rs b/nodedb-test-support/src/fail_point.rs new file mode 100644 index 000000000..9ae616d2c --- /dev/null +++ b/nodedb-test-support/src/fail_point.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The fail-point test API. +//! +//! An in-process cluster shares one fail-point registry across its nodes. +//! [`FailGuard::for_node`] arms a point on one node, and +//! [`FailGuard::install`] arms it on every node. + +pub use nodedb_types::fail_point::{FailAction, FailGuard, FailScope}; diff --git a/nodedb-test-support/src/lib.rs b/nodedb-test-support/src/lib.rs index f2c785607..4c2aef699 100644 --- a/nodedb-test-support/src/lib.rs +++ b/nodedb-test-support/src/lib.rs @@ -7,6 +7,8 @@ pub mod booted_state; pub mod catalog_fixtures; pub mod cluster_harness; pub mod core_loop_runner; +#[cfg(feature = "failpoints")] +pub mod fail_point; pub mod ilp_client; pub mod insert_returning_engines; pub mod jwks_fixture; diff --git a/nodedb-test-support/src/pgwire_harness/multicore.rs b/nodedb-test-support/src/pgwire_harness/multicore.rs index 24b04b9bb..85e1add27 100644 --- a/nodedb-test-support/src/pgwire_harness/multicore.rs +++ b/nodedb-test-support/src/pgwire_harness/multicore.rs @@ -68,6 +68,7 @@ impl TestServer { let core_handle = crate::core_loop_runner::spawn_core_loop(crate::core_loop_runner::CoreLoopSpawn { idx, + node_id: shared.node_id, num_cores, data_side, core_dir: dir.path().to_path_buf(), diff --git a/nodedb-test-support/src/pgwire_harness/restart.rs b/nodedb-test-support/src/pgwire_harness/restart.rs index 860b419a6..9d270e9fa 100644 --- a/nodedb-test-support/src/pgwire_harness/restart.rs +++ b/nodedb-test-support/src/pgwire_harness/restart.rs @@ -247,6 +247,7 @@ impl TestServer { let core_handle = crate::core_loop_runner::spawn_core_loop(crate::core_loop_runner::CoreLoopSpawn { idx, + node_id: shared.node_id, // Single-core harness (`Dispatcher::new(1, ..)`). num_cores: 1, data_side, diff --git a/nodedb-test-support/src/pgwire_harness/start.rs b/nodedb-test-support/src/pgwire_harness/start.rs index 86c27cdcd..c3ce2e977 100644 --- a/nodedb-test-support/src/pgwire_harness/start.rs +++ b/nodedb-test-support/src/pgwire_harness/start.rs @@ -268,6 +268,7 @@ impl TestServer { let core_handle = crate::core_loop_runner::spawn_core_loop(crate::core_loop_runner::CoreLoopSpawn { idx, + node_id: shared.node_id, // Single-core harness (`Dispatcher::new(1, ..)`). num_cores: 1, data_side, diff --git a/nodedb-types/src/fail_point.rs b/nodedb-types/src/fail_point.rs deleted file mode 100644 index 88948da0e..000000000 --- a/nodedb-types/src/fail_point.rs +++ /dev/null @@ -1,347 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Minimal feature-gated fail-point framework for crash-injection tests. -//! -//! When the `failpoints` Cargo feature is OFF (the default and what release -//! builds compile with), `fail_point!(name)` expands to nothing — zero -//! runtime cost. -//! -//! When the feature is ON, each invocation looks up `name` in a process-wide -//! registry. If a [`FailAction`] has been installed for that name, the action -//! fires: -//! - `Panic`: panic immediately with a message naming the fail point. -//! - `Sleep(d)`: sleep for `d`, useful for race-condition tests. -//! - `Fail(detail)`: make the injected call site return an error. Only -//! [`fail_point_err!`] can honour this — the call site supplies the -//! mapping into its own error type, so no crate has to know about -//! anyone else's. -//! - `WaitForFile(path)`: park the call site until `path` exists, so a test -//! releases it at the moment it chooses. Only an async call site can -//! honour this: it awaits the file with the crate's own timer and parks -//! only its own task. A synchronous call site refuses it. -//! -//! The framework is deliberately tiny — no fail-rs dep, no parsing of env -//! vars, no list of probabilities. Tests install actions explicitly via -//! `set` / `clear` and must clean up in their own teardown (or use -//! `FailGuard` for RAII cleanup). -//! -//! Feature gating is per-crate: the macros expand under the *calling* crate's -//! `failpoints` feature, so every crate that injects a fail point declares its -//! own `failpoints = ["nodedb-types/failpoints"]`. - -#[cfg(feature = "failpoints")] -mod imp { - use std::collections::HashMap; - use std::sync::{LazyLock, Mutex}; - use std::time::Duration; - - /// Action a fail point performs when triggered. - #[derive(Debug, Clone)] - pub enum FailAction { - /// Panic with a message that names the fail-point. - Panic, - /// Kill the whole process immediately via `abort()`. - /// - /// `Panic` only unwinds the current task, which a supervising runtime - /// absorbs — no use for simulating a crash in a spawned server. This - /// is the action a process-kill harness arms. - Abort, - /// Sleep for the given duration; execution then continues normally. - Sleep(Duration), - /// Return an error from the injected call site, carrying this detail. - /// Ignored by bare `fail_point!` — use `fail_point_err!`. - Fail(String), - /// Park the call site until this file exists. Only an async call site - /// that looks the action up with [`lookup`] can honour it. - WaitForFile(std::path::PathBuf), - } - - /// Environment variable read once, the first time any fail point is - /// evaluated. Lets a test arm injections in a *spawned* process — the - /// in-process `set` API cannot reach a server the test only supervises. - /// - /// Format: comma-separated `name=action`, where action is `panic`, - /// `abort`, `sleep()`, `fail()`, or `wait_file()`. - /// For example: - /// `NODEDB_FAILPOINTS='checkpoint::after_marker_before_truncate=panic'` - pub const FAILPOINTS_ENV: &str = "NODEDB_FAILPOINTS"; - - static REGISTRY: LazyLock>> = - LazyLock::new(|| Mutex::new(parse_env(std::env::var(FAILPOINTS_ENV).ok().as_deref()))); - - pub(super) fn parse_env(spec: Option<&str>) -> HashMap { - let mut actions = HashMap::new(); - let Some(spec) = spec else { - return actions; - }; - for entry in spec.split(',').map(str::trim).filter(|e| !e.is_empty()) { - let Some((name, action)) = entry.split_once('=') else { - panic!("{FAILPOINTS_ENV} entry {entry:?} is not `name=action`"); - }; - let action = match action.trim() { - "panic" => FailAction::Panic, - "abort" => FailAction::Abort, - rest if rest.starts_with("sleep(") && rest.ends_with(')') => { - let millis = - rest["sleep(".len()..rest.len() - 1] - .parse() - .unwrap_or_else(|_| { - panic!("{FAILPOINTS_ENV} entry {entry:?} has a non-numeric sleep") - }); - FailAction::Sleep(Duration::from_millis(millis)) - } - rest if rest.starts_with("fail(") && rest.ends_with(')') => { - FailAction::Fail(rest["fail(".len()..rest.len() - 1].to_string()) - } - rest if rest.starts_with("wait_file(") && rest.ends_with(')') => { - FailAction::WaitForFile(std::path::PathBuf::from( - &rest["wait_file(".len()..rest.len() - 1], - )) - } - other => panic!("{FAILPOINTS_ENV} entry {entry:?} has unknown action {other:?}"), - }; - actions.insert(name.trim().to_string(), action); - } - actions - } - - /// Install an action for a named fail point. Subsequent - /// `fail_point!(name)` invocations will fire the action. - pub fn set(name: &str, action: FailAction) { - REGISTRY - .lock() - .unwrap_or_else(|p| p.into_inner()) - .insert(name.to_string(), action); - } - - /// Remove any installed action for the named fail point. - pub fn clear(name: &str) { - REGISTRY - .lock() - .unwrap_or_else(|p| p.into_inner()) - .remove(name); - } - - /// Look up the installed action for a fail point (None if not set). - pub fn lookup(name: &str) -> Option { - REGISTRY - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get(name) - .cloned() - } - - /// Evaluate the installed action for a fail point. Used by the - /// `fail_point!` macro; not intended to be called directly. - pub fn eval(name: &str) { - if let Some(action) = lookup(name) { - match action { - FailAction::Panic => panic!("fail_point fired: {name}"), - FailAction::Abort => abort(name), - FailAction::Sleep(d) => std::thread::sleep(d), - // A caller that cannot return an error must not silently - // swallow the injection: an armed failpoint that does nothing - // makes a test pass for the wrong reason. - FailAction::Fail(detail) => { - panic!( - "fail_point {name} installed Fail({detail}) but the call site cannot return an error — use fail_point_err!" - ) - } - // Blocking the thread would stall every task that shares it. - FailAction::WaitForFile(path) => panic!( - "fail_point {name} installed WaitForFile({}) but the call site is \ - synchronous — only an async call site can park", - path.display() - ), - } - } - } - - /// Kill the process at a fail point. Flushes the reason first — an - /// unexplained SIGABRT in a test log is indistinguishable from a real bug. - fn abort(name: &str) -> ! { - eprintln!("fail_point aborting process: {name}"); - std::process::abort() - } - - /// Evaluate a fail point that can return an error, yielding the detail - /// string when the installed action is [`FailAction::Fail`]. Used by the - /// `fail_point_err!` macro; not intended to be called directly. - pub fn eval_fail(name: &str) -> Option { - match lookup(name) { - Some(FailAction::Fail(detail)) => Some(detail), - Some(FailAction::Panic) => panic!("fail_point fired: {name}"), - Some(FailAction::Abort) => abort(name), - Some(FailAction::Sleep(d)) => { - std::thread::sleep(d); - None - } - Some(FailAction::WaitForFile(path)) => panic!( - "fail_point {name} installed WaitForFile({}) but the call site is synchronous \ - — only an async call site can park", - path.display() - ), - None => None, - } - } - - /// RAII guard that clears a fail point on drop. Use to keep tests - /// from leaking installed actions across cases. - pub struct FailGuard { - name: String, - } - - impl FailGuard { - pub fn install(name: &str, action: FailAction) -> Self { - set(name, action); - Self { - name: name.to_string(), - } - } - - /// Arm a fail point to return an error carrying `detail`. - pub fn fail(name: &str, detail: &str) -> Self { - Self::install(name, FailAction::Fail(detail.to_string())) - } - } - - impl Drop for FailGuard { - fn drop(&mut self) { - clear(&self.name); - } - } -} - -#[cfg(feature = "failpoints")] -pub use imp::{FAILPOINTS_ENV, FailAction, FailGuard, clear, eval, eval_fail, lookup, set}; - -/// Inject a fail point. Expands to nothing without the `failpoints` feature. -/// -/// Usage in production code: -/// `nodedb_types::fail_point!("calvin_static::during_overlay_stage");` -/// -/// Tests opt in by enabling the feature and installing actions: -/// `fail_point::set("calvin_static::during_overlay_stage", -/// fail_point::FailAction::Panic);` -#[macro_export] -macro_rules! fail_point { - ($name:expr) => { - #[cfg(feature = "failpoints")] - $crate::fail_point::eval($name); - }; -} - -/// Inject a fail point that can abort the enclosing function with an error. -/// -/// `$map` receives the detail string installed with [`FailAction::Fail`] and -/// returns the error value to propagate, so each crate injects its own error -/// type without the framework knowing about it: -/// -/// ```ignore -/// fail_point_err!("wal::flush_out_of_space", |_| WalError::OutOfSpace { -/// context: "WAL segment append (failpoint)", -/// }); -/// ``` -/// -/// Expands to nothing without the `failpoints` feature. -#[macro_export] -macro_rules! fail_point_err { - ($name:expr, $map:expr) => { - #[cfg(feature = "failpoints")] - if let Some(detail) = $crate::fail_point::eval_fail($name) { - return Err(($map)(detail)); - } - }; -} - -#[cfg(all(test, feature = "failpoints"))] -mod tests { - use super::*; - - #[test] - fn unset_fail_point_is_noop() { - eval("nodedb::test::unset"); - assert_eq!(eval_fail("nodedb::test::unset"), None); - } - - #[test] - #[should_panic(expected = "fail_point fired: nodedb::test::panic_target")] - fn set_panic_fires() { - let _g = FailGuard::install("nodedb::test::panic_target", FailAction::Panic); - eval("nodedb::test::panic_target"); - } - - #[test] - fn fail_action_yields_its_detail() { - let _g = FailGuard::fail("nodedb::test::fail_target", "disk full"); - assert_eq!( - eval_fail("nodedb::test::fail_target"), - Some("disk full".to_string()) - ); - } - - #[test] - #[should_panic(expected = "cannot return an error")] - fn fail_action_at_an_infallible_call_site_is_loud() { - let _g = FailGuard::fail("nodedb::test::fail_at_infallible", "nope"); - eval("nodedb::test::fail_at_infallible"); - } - - #[test] - fn env_spec_parses_every_action() { - let actions = super::imp::parse_env(Some( - "a::panic=panic, b::sleep=sleep(25), c::fail=fail(disk full)", - )); - assert!(matches!(actions.get("a::panic"), Some(FailAction::Panic))); - assert!(matches!( - actions.get("b::sleep"), - Some(FailAction::Sleep(d)) if *d == std::time::Duration::from_millis(25) - )); - assert!(matches!( - actions.get("c::fail"), - Some(FailAction::Fail(detail)) if detail == "disk full" - )); - } - - #[test] - fn env_spec_parses_a_file_gate() { - let actions = super::imp::parse_env(Some("d::gate=wait_file(/tmp/release-d)")); - assert!(matches!( - actions.get("d::gate"), - Some(FailAction::WaitForFile(path)) if path == std::path::Path::new("/tmp/release-d") - )); - } - - #[test] - #[should_panic(expected = "only an async call site can park")] - fn a_file_gate_at_a_synchronous_call_site_is_loud() { - let _g = FailGuard::install( - "nodedb::test::gate_at_sync", - FailAction::WaitForFile(std::path::PathBuf::from("/nonexistent")), - ); - eval("nodedb::test::gate_at_sync"); - } - - #[test] - fn empty_env_spec_arms_nothing() { - assert!(super::imp::parse_env(None).is_empty()); - assert!(super::imp::parse_env(Some("")).is_empty()); - } - - #[test] - #[should_panic(expected = "unknown action")] - fn malformed_env_spec_is_loud() { - // A typo that silently armed nothing would make a crash test pass - // without ever injecting the crash. - super::imp::parse_env(Some("a::b=explode")); - } - - #[test] - fn fail_guard_clears_on_drop() { - { - let _g = FailGuard::install("nodedb::test::guard_clear", FailAction::Panic); - assert!(lookup("nodedb::test::guard_clear").is_some()); - } - assert!(lookup("nodedb::test::guard_clear").is_none()); - } -} diff --git a/nodedb-types/src/fail_point/action.rs b/nodedb-types/src/fail_point/action.rs new file mode 100644 index 000000000..08c9d9069 --- /dev/null +++ b/nodedb-types/src/fail_point/action.rs @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! What an armed fail point does when it fires. + +use std::time::Duration; + +/// Action a fail point performs when triggered. +#[derive(Debug, Clone)] +pub enum FailAction { + /// Panic with a message that names the fail-point. + Panic, + /// Kill the whole process immediately via `abort()`. + /// + /// `Panic` only unwinds the current task, which a supervising runtime + /// absorbs — no use for simulating a crash in a spawned server. This + /// is the action a process-kill harness arms. + Abort, + /// Sleep for the given duration; execution then continues normally. + Sleep(Duration), + /// Return an error from the injected call site, carrying this detail. + /// Ignored by bare `fail_point!` — use `fail_point_err!`. + Fail(String), + /// Park the call site until this file exists. Only an async call site + /// that looks the action up with [`super::lookup`] can honour it. + WaitForFile(std::path::PathBuf), +} diff --git a/nodedb-types/src/fail_point/env.rs b/nodedb-types/src/fail_point/env.rs new file mode 100644 index 000000000..67de29130 --- /dev/null +++ b/nodedb-types/src/fail_point/env.rs @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The environment spec that arms fail points in a spawned process. + +use std::time::Duration; + +use super::action::FailAction; +use super::registry::ArmedActions; +use super::scope::FailScope; + +/// Environment variable read once, the first time any fail point is +/// evaluated. Lets a test arm injections in a *spawned* process — the +/// in-process `set` API cannot reach a server the test only supervises. +/// +/// Format: comma-separated `name=action` or `name@node=action`. The +/// first arms every node, the second only node `N`. The action is `panic`, +/// `abort`, `sleep()`, `fail()`, or `wait_file()`. +/// For example: +/// `NODEDB_FAILPOINTS='checkpoint::after_marker_before_truncate=panic'` +pub const FAILPOINTS_ENV: &str = "NODEDB_FAILPOINTS"; + +/// Parse a [`FAILPOINTS_ENV`] spec. Panics on a malformed entry: a typo that +/// armed nothing would let a crash test pass without injecting the crash. +pub(crate) fn parse_env(spec: Option<&str>) -> ArmedActions { + let mut actions = ArmedActions::new(); + let Some(spec) = spec else { + return actions; + }; + for entry in spec.split(',').map(str::trim).filter(|e| !e.is_empty()) { + let Some((target, action)) = entry.split_once('=') else { + panic!( + "{FAILPOINTS_ENV} entry {entry:?} is not `name=action` or `name@node=action`" + ); + }; + let (name, scope) = parse_target(entry, target.trim()); + let action = parse_action(entry, action.trim()); + actions + .entry(name.to_string()) + .or_default() + .insert(scope, action); + } + actions +} + +/// Split `name@node` into the name and `Node(N)`. A target with no `@` +/// is scoped to every node. +fn parse_target<'a>(entry: &str, target: &'a str) -> (&'a str, FailScope) { + let (name, scope) = match target.rsplit_once('@') { + None => (target, FailScope::Any), + Some((name, scope)) => { + let node = scope + .trim() + .strip_prefix("node") + .and_then(|id| id.parse().ok()) + .unwrap_or_else(|| { + panic!("{FAILPOINTS_ENV} entry {entry:?} has scope {scope:?}, not `node`") + }); + (name.trim(), FailScope::Node(node)) + } + }; + if name.is_empty() { + panic!("{FAILPOINTS_ENV} entry {entry:?} names no fail point"); + } + (name, scope) +} + +fn parse_action(entry: &str, action: &str) -> FailAction { + match action { + "panic" => FailAction::Panic, + "abort" => FailAction::Abort, + rest if rest.starts_with("sleep(") && rest.ends_with(')') => { + let millis = rest["sleep(".len()..rest.len() - 1] + .parse() + .unwrap_or_else(|_| { + panic!("{FAILPOINTS_ENV} entry {entry:?} has a non-numeric sleep") + }); + FailAction::Sleep(Duration::from_millis(millis)) + } + rest if rest.starts_with("fail(") && rest.ends_with(')') => { + FailAction::Fail(rest["fail(".len()..rest.len() - 1].to_string()) + } + rest if rest.starts_with("wait_file(") && rest.ends_with(')') => FailAction::WaitForFile( + std::path::PathBuf::from(&rest["wait_file(".len()..rest.len() - 1]), + ), + other => panic!("{FAILPOINTS_ENV} entry {entry:?} has unknown action {other:?}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn armed<'a>( + actions: &'a ArmedActions, + scope: FailScope, + name: &str, + ) -> Option<&'a FailAction> { + actions.get(name).and_then(|scopes| scopes.get(&scope)) + } + + #[test] + fn env_spec_parses_every_action() { + let actions = parse_env(Some( + "a::panic=panic, b::sleep=sleep(25), c::fail=fail(disk full)", + )); + assert!(matches!( + armed(&actions, FailScope::Any, "a::panic"), + Some(FailAction::Panic) + )); + assert!(matches!( + armed(&actions, FailScope::Any, "b::sleep"), + Some(FailAction::Sleep(d)) if *d == Duration::from_millis(25) + )); + assert!(matches!( + armed(&actions, FailScope::Any, "c::fail"), + Some(FailAction::Fail(detail)) if detail == "disk full" + )); + } + + #[test] + fn env_spec_parses_a_file_gate() { + let actions = parse_env(Some("d::gate=wait_file(/tmp/release-d)")); + assert!(matches!( + armed(&actions, FailScope::Any, "d::gate"), + Some(FailAction::WaitForFile(path)) if path == std::path::Path::new("/tmp/release-d") + )); + } + + #[test] + fn env_spec_scopes_an_entry_to_one_node() { + let actions = parse_env(Some("e::hold@node3=abort, e::hold=panic")); + assert!(matches!( + armed(&actions, FailScope::Node(3), "e::hold"), + Some(FailAction::Abort) + )); + assert!(matches!( + armed(&actions, FailScope::Any, "e::hold"), + Some(FailAction::Panic) + )); + assert!(armed(&actions, FailScope::Node(2), "e::hold").is_none()); + } + + #[test] + fn a_node_scope_leaves_the_action_path_alone() { + let actions = parse_env(Some("f::gate@node1=wait_file(/tmp/a@b=c)")); + assert!(matches!( + armed(&actions, FailScope::Node(1), "f::gate"), + Some(FailAction::WaitForFile(path)) if path == std::path::Path::new("/tmp/a@b=c") + )); + } + + #[test] + fn empty_env_spec_arms_nothing() { + assert!(parse_env(None).is_empty()); + assert!(parse_env(Some("")).is_empty()); + } + + #[test] + #[should_panic(expected = "unknown action")] + fn malformed_env_spec_is_loud() { + parse_env(Some("a::b=explode")); + } + + #[test] + #[should_panic(expected = "not `node`")] + fn a_scope_that_is_not_a_node_is_loud() { + parse_env(Some("a::b@core1=panic")); + } + + #[test] + #[should_panic(expected = "not `node`")] + fn a_node_scope_without_an_id_is_loud() { + parse_env(Some("a::b@node=panic")); + } + + #[test] + #[should_panic(expected = "names no fail point")] + fn a_scope_with_no_name_is_loud() { + parse_env(Some("@node1=panic")); + } +} diff --git a/nodedb-types/src/fail_point/guard.rs b/nodedb-types/src/fail_point/guard.rs new file mode 100644 index 000000000..56daf936f --- /dev/null +++ b/nodedb-types/src/fail_point/guard.rs @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! RAII arming of a fail point for one test. + +use super::action::FailAction; +use super::registry::{clear, set}; +use super::scope::FailScope; + +/// Arms a fail point and disarms it on drop. Use to keep tests from +/// leaking armed actions across cases. +pub struct FailGuard { + scope: FailScope, + name: String, +} + +impl FailGuard { + /// Arm `name` on every node. + pub fn install(name: &str, action: FailAction) -> Self { + Self::arm(FailScope::Any, name, action) + } + + /// Arm `name` on node `node_id` only. Every other node runs past it. + pub fn for_node(node_id: u64, name: &str, action: FailAction) -> Self { + Self::arm(FailScope::Node(node_id), name, action) + } + + /// Arm `name` on every node to return an error carrying `detail`. + pub fn fail(name: &str, detail: &str) -> Self { + Self::install(name, FailAction::Fail(detail.to_string())) + } + + fn arm(scope: FailScope, name: &str, action: FailAction) -> Self { + set(scope, name, action); + Self { + scope, + name: name.to_string(), + } + } +} + +impl Drop for FailGuard { + fn drop(&mut self) { + clear(self.scope, &self.name); + } +} diff --git a/nodedb-types/src/fail_point/macros.rs b/nodedb-types/src/fail_point/macros.rs new file mode 100644 index 000000000..cd3be8550 --- /dev/null +++ b/nodedb-types/src/fail_point/macros.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The `fail_point!` and `fail_point_err!` injection macros. +//! +//! Each takes the scope the point is evaluated in as its first argument. A +//! call site with a node identity passes `FailScope::Node(id)`. The form +//! without a scope evaluates in `FailScope::Any`, for code with no node +//! identity. Without the calling crate's `failpoints` feature, a scoped form +//! only type-checks its scope and evaluates no name. + +/// Inject a fail point. Does nothing at runtime without the `failpoints` +/// feature. +/// +/// Usage in production code: +/// `nodedb_types::fail_point!(scope, "calvin_static::during_overlay_stage");` +/// `nodedb_types::fail_point!("wal::roll_before_seal");` (no node identity) +/// +/// Tests opt in by enabling the feature and arming actions: +/// `FailGuard::for_node(node_id, "calvin_static::during_overlay_stage", +/// FailAction::Panic);` +#[macro_export] +macro_rules! fail_point { + ($name:expr) => { + #[cfg(feature = "failpoints")] + $crate::fail_point::eval($crate::fail_point::FailScope::Any, $name); + }; + ($scope:expr, $name:expr) => { + #[cfg(feature = "failpoints")] + $crate::fail_point::eval($scope, $name); + #[cfg(not(feature = "failpoints"))] + let _: $crate::fail_point::FailScope = $scope; + }; +} + +/// Inject a fail point that can abort the enclosing function with an error. +/// +/// `$map` receives the detail string armed with `FailAction::Fail` and +/// returns the error value to propagate, so each crate injects its own error +/// type without the framework knowing about it: +/// +/// ```ignore +/// fail_point_err!("wal::flush_out_of_space", |_| WalError::OutOfSpace { +/// context: "WAL segment append (failpoint)", +/// }); +/// fail_point_err!(FailScope::Node(node_id), "cut_floor::before_persist", |detail| { +/// Error::Internal { detail } +/// }); +/// ``` +/// +/// Does nothing at runtime without the `failpoints` feature. +#[macro_export] +macro_rules! fail_point_err { + ($name:expr, $map:expr) => { + #[cfg(feature = "failpoints")] + if let Some(detail) = + $crate::fail_point::eval_fail($crate::fail_point::FailScope::Any, $name) + { + return Err(($map)(detail)); + } + }; + ($scope:expr, $name:expr, $map:expr) => { + #[cfg(feature = "failpoints")] + if let Some(detail) = $crate::fail_point::eval_fail($scope, $name) { + return Err(($map)(detail)); + } + #[cfg(not(feature = "failpoints"))] + let _: $crate::fail_point::FailScope = $scope; + }; +} diff --git a/nodedb-types/src/fail_point/mod.rs b/nodedb-types/src/fail_point/mod.rs new file mode 100644 index 000000000..84db4f922 --- /dev/null +++ b/nodedb-types/src/fail_point/mod.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Minimal feature-gated fail-point framework for crash-injection tests. +//! +//! When the `failpoints` Cargo feature is OFF (the default and what release +//! builds compile with), `fail_point!` expands to nothing at runtime. +//! +//! When the feature is ON, each invocation looks up its scope and name in a +//! process-wide registry. A point evaluated with scope `Node(n)` fires the +//! action armed for `(Node(n), name)`, else the one armed for `(Any, name)`. +//! A point evaluated with scope `Any` fires only the `(Any, name)` action. +//! The actions are listed on [`FailAction`]. +//! +//! An in-process cluster test runs several nodes in one process, so they +//! share the registry. The scope arms one node's point and leaves the +//! others running. +//! +//! Tests arm actions with `set` / `clear` or the RAII [`FailGuard`]. A +//! spawned server reads its actions once from [`FAILPOINTS_ENV`]. +//! +//! Feature gating is per-crate: the macros expand under the *calling* crate's +//! `failpoints` feature, so every crate that injects a fail point declares its +//! own `failpoints = ["nodedb-types/failpoints"]`. + +#[cfg(feature = "failpoints")] +pub mod action; +#[cfg(feature = "failpoints")] +pub mod env; +#[cfg(feature = "failpoints")] +pub mod guard; +pub mod macros; +#[cfg(feature = "failpoints")] +pub mod registry; +pub mod scope; + +#[cfg(feature = "failpoints")] +pub use action::FailAction; +#[cfg(feature = "failpoints")] +pub use env::FAILPOINTS_ENV; +#[cfg(feature = "failpoints")] +pub use guard::FailGuard; +#[cfg(feature = "failpoints")] +pub use registry::{clear, eval, eval_fail, lookup, set}; +pub use scope::FailScope; diff --git a/nodedb-types/src/fail_point/registry.rs b/nodedb-types/src/fail_point/registry.rs new file mode 100644 index 000000000..37e38c0c4 --- /dev/null +++ b/nodedb-types/src/fail_point/registry.rs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The process-wide registry of armed fail points, and their evaluation. + +use std::collections::HashMap; +use std::sync::{LazyLock, Mutex, MutexGuard}; + +use super::action::FailAction; +use super::env::{FAILPOINTS_ENV, parse_env}; +use super::scope::FailScope; + +/// Armed actions by fail point name, then by scope. +pub(crate) type ArmedActions = HashMap>; + +static REGISTRY: LazyLock> = + LazyLock::new(|| Mutex::new(parse_env(std::env::var(FAILPOINTS_ENV).ok().as_deref()))); + +fn registry() -> MutexGuard<'static, ArmedActions> { + REGISTRY.lock().unwrap_or_else(|p| p.into_inner()) +} + +/// Arm `name` for `scope`. A later point evaluated in a matching scope fires +/// `action`. +pub fn set(scope: FailScope, name: &str, action: FailAction) { + registry() + .entry(name.to_string()) + .or_default() + .insert(scope, action); +} + +/// Disarm `name` for `scope`. Actions armed for other scopes stay. +pub fn clear(scope: FailScope, name: &str) { + let mut armed = registry(); + if let Some(scopes) = armed.get_mut(name) { + scopes.remove(&scope); + if scopes.is_empty() { + armed.remove(name); + } + } +} + +/// The action a point `name` evaluated in `scope` fires, if any. +/// +/// `Node(n)` takes the action armed for `Node(n)`, else the one armed for +/// `Any`. `Any` takes only the `Any` action. +pub fn lookup(scope: FailScope, name: &str) -> Option { + let armed = registry(); + let scopes = armed.get(name)?; + scopes + .get(&scope) + .or_else(|| scopes.get(&FailScope::Any)) + .cloned() +} + +/// Evaluate point `name` in `scope`. Used by the `fail_point!` macro; not +/// intended to be called directly. +pub fn eval(scope: FailScope, name: &str) { + if let Some(action) = lookup(scope, name) { + match action { + FailAction::Panic => panic!("fail_point fired: {name} ({scope})"), + FailAction::Abort => abort(scope, name), + FailAction::Sleep(d) => std::thread::sleep(d), + // A caller that cannot return an error must not silently + // swallow the injection: an armed failpoint that does nothing + // makes a test pass for the wrong reason. + FailAction::Fail(detail) => panic!( + "fail_point {name} ({scope}) installed Fail({detail}) but the call site cannot \ + return an error — use fail_point_err!" + ), + // Blocking the thread would stall every task that shares it. + FailAction::WaitForFile(path) => panic!( + "fail_point {name} ({scope}) installed WaitForFile({}) but the call site is \ + synchronous — only an async call site can park", + path.display() + ), + } + } +} + +/// Evaluate point `name` in `scope` at a call site that can return an error. +/// Yields the detail when the action is [`FailAction::Fail`]. Used by the +/// `fail_point_err!` macro; not intended to be called directly. +pub fn eval_fail(scope: FailScope, name: &str) -> Option { + match lookup(scope, name) { + Some(FailAction::Fail(detail)) => Some(detail), + Some(FailAction::Panic) => panic!("fail_point fired: {name} ({scope})"), + Some(FailAction::Abort) => abort(scope, name), + Some(FailAction::Sleep(d)) => { + std::thread::sleep(d); + None + } + Some(FailAction::WaitForFile(path)) => panic!( + "fail_point {name} ({scope}) installed WaitForFile({}) but the call site is \ + synchronous — only an async call site can park", + path.display() + ), + None => None, + } +} + +/// Kill the process at a fail point. Flushes the reason first — an +/// unexplained SIGABRT in a test log is indistinguishable from a real bug. +fn abort(scope: FailScope, name: &str) -> ! { + eprintln!("fail_point aborting process: {name} ({scope})"); + std::process::abort() +} + +#[cfg(test)] +mod tests { + use super::super::guard::FailGuard; + use super::*; + + #[test] + fn unset_fail_point_is_noop() { + eval(FailScope::Any, "nodedb::test::unset"); + assert_eq!(eval_fail(FailScope::Node(1), "nodedb::test::unset"), None); + } + + #[test] + #[should_panic(expected = "fail_point fired: nodedb::test::panic_target")] + fn set_panic_fires() { + let _g = FailGuard::install("nodedb::test::panic_target", FailAction::Panic); + eval(FailScope::Any, "nodedb::test::panic_target"); + } + + #[test] + fn fail_action_yields_its_detail() { + let _g = FailGuard::fail("nodedb::test::fail_target", "disk full"); + assert_eq!( + eval_fail(FailScope::Any, "nodedb::test::fail_target"), + Some("disk full".to_string()) + ); + } + + #[test] + #[should_panic(expected = "cannot return an error")] + fn fail_action_at_an_infallible_call_site_is_loud() { + let _g = FailGuard::fail("nodedb::test::fail_at_infallible", "nope"); + eval(FailScope::Any, "nodedb::test::fail_at_infallible"); + } + + #[test] + #[should_panic(expected = "only an async call site can park")] + fn a_file_gate_at_a_synchronous_call_site_is_loud() { + let _g = FailGuard::install( + "nodedb::test::gate_at_sync", + FailAction::WaitForFile(std::path::PathBuf::from("/nonexistent")), + ); + eval(FailScope::Any, "nodedb::test::gate_at_sync"); + } + + #[test] + fn an_any_action_fires_on_every_node() { + let _g = FailGuard::fail("nodedb::test::any_scope", "every node"); + for scope in [FailScope::Any, FailScope::Node(1), FailScope::Node(2)] { + assert_eq!( + eval_fail(scope, "nodedb::test::any_scope"), + Some("every node".to_string()), + "{scope}" + ); + } + } + + #[test] + fn a_node_action_fires_only_on_its_node() { + let _g = FailGuard::for_node( + 2, + "nodedb::test::node_scope", + FailAction::Fail("node two".to_string()), + ); + assert_eq!( + eval_fail(FailScope::Node(2), "nodedb::test::node_scope"), + Some("node two".to_string()) + ); + assert_eq!( + eval_fail(FailScope::Node(1), "nodedb::test::node_scope"), + None + ); + assert_eq!(eval_fail(FailScope::Any, "nodedb::test::node_scope"), None); + } + + #[test] + fn a_node_action_wins_over_an_any_action() { + let _any = FailGuard::fail("nodedb::test::both_scopes", "any"); + let _node = FailGuard::for_node( + 3, + "nodedb::test::both_scopes", + FailAction::Fail("node three".to_string()), + ); + assert_eq!( + eval_fail(FailScope::Node(3), "nodedb::test::both_scopes"), + Some("node three".to_string()) + ); + assert_eq!( + eval_fail(FailScope::Node(4), "nodedb::test::both_scopes"), + Some("any".to_string()) + ); + } + + #[test] + fn clearing_one_scope_keeps_the_other() { + let _any = FailGuard::fail("nodedb::test::clear_one", "any"); + { + let _node = FailGuard::for_node( + 5, + "nodedb::test::clear_one", + FailAction::Fail("node five".to_string()), + ); + } + assert_eq!( + eval_fail(FailScope::Node(5), "nodedb::test::clear_one"), + Some("any".to_string()) + ); + } + + #[test] + fn fail_guard_clears_on_drop() { + { + let _g = FailGuard::install("nodedb::test::guard_clear", FailAction::Panic); + assert!(lookup(FailScope::Any, "nodedb::test::guard_clear").is_some()); + } + assert!(lookup(FailScope::Any, "nodedb::test::guard_clear").is_none()); + } +} diff --git a/nodedb-types/src/fail_point/scope.rs b/nodedb-types/src/fail_point/scope.rs new file mode 100644 index 000000000..78c8e5aed --- /dev/null +++ b/nodedb-types/src/fail_point/scope.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The node scope of a fail point. + +use std::fmt; + +/// The nodes an armed fail point applies to, and the node a point is +/// evaluated on. +/// +/// Compiled without the `failpoints` feature too, so a call site can name +/// its scope in every build. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FailScope { + /// Every node. Code with no node identity evaluates its points here. + Any, + /// The node with this id. + Node(u64), +} + +impl fmt::Display for FailScope { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Any => f.write_str("any node"), + Self::Node(id) => write!(f, "node {id}"), + } + } +} diff --git a/nodedb-wal/src/uring_writer.rs b/nodedb-wal/src/uring_writer.rs index 6affceb8a..a3ce1827a 100644 --- a/nodedb-wal/src/uring_writer.rs +++ b/nodedb-wal/src/uring_writer.rs @@ -320,7 +320,10 @@ impl UringWriter { fn submit_and_wait_fsync(&mut self) -> Result<()> { // Crash injection: the kernel reports a writeback error at the fsync. #[cfg(feature = "failpoints")] - if let Some(detail) = nodedb_types::fail_point::eval_fail("wal::fsync_failure") { + if let Some(detail) = nodedb_types::fail_point::eval_fail( + nodedb_types::fail_point::FailScope::Any, + "wal::fsync_failure", + ) { return Err(self.durability.poison(format!( "io_uring fsync failed: failpoint wal::fsync_failure: {detail}" ))); diff --git a/nodedb-wal/src/writer/durability.rs b/nodedb-wal/src/writer/durability.rs index 72ac9f8a3..2a74541f2 100644 --- a/nodedb-wal/src/writer/durability.rs +++ b/nodedb-wal/src/writer/durability.rs @@ -98,7 +98,10 @@ pub(crate) fn fsync_and_track(file: &File, state: &mut DurabilityState) -> Resul // Crash injection: the kernel reports a writeback error at the fsync. // Everything already handed to the page cache is unrecoverable from here. #[cfg(feature = "failpoints")] - if let Some(detail) = nodedb_types::fail_point::eval_fail("wal::fsync_failure") { + if let Some(detail) = nodedb_types::fail_point::eval_fail( + nodedb_types::fail_point::FailScope::Any, + "wal::fsync_failure", + ) { return Err(state.poison(format!( "fsync failed: failpoint wal::fsync_failure: {detail}" ))); diff --git a/nodedb/src/bootstrap/data_plane.rs b/nodedb/src/bootstrap/data_plane.rs index ed877955f..3b29aabba 100644 --- a/nodedb/src/bootstrap/data_plane.rs +++ b/nodedb/src/bootstrap/data_plane.rs @@ -327,6 +327,7 @@ pub fn spawn_data_plane_cores( columnar_schema_seed, } = resources; let num_cores = config.server.data_plane_cores; + let node_id = crate::control::cluster::configured_node_id(config); let compaction_cfg = CoreCompactionConfig { interval: config.checkpoint.compaction_interval(), tombstone_threshold: config.checkpoint.compaction_tombstone_threshold, @@ -347,6 +348,7 @@ pub fn spawn_data_plane_cores( let (replay_done_tx, replay_done_rx) = tokio::sync::oneshot::channel(); let (handle, notifier) = spawn_core(SpawnCoreParams { core_id, + node_id, request_rx: data_side.request_rx, response_tx: data_side.response_tx, data_dir: &config.server.data_dir, diff --git a/nodedb/src/control/backup/cut_capture/collect.rs b/nodedb/src/control/backup/cut_capture/collect.rs index fca3dcb87..61f916827 100644 --- a/nodedb/src/control/backup/cut_capture/collect.rs +++ b/nodedb/src/control/backup/cut_capture/collect.rs @@ -101,7 +101,11 @@ pub(crate) async fn capture_database( // A test parks the backup here: every capture is taken, so a write from // now on lies above the cut. #[cfg(feature = "failpoints")] - crate::control::fail_gate::wait("backup::database::after_cut").await; + crate::control::fail_gate::wait( + crate::fail_point::FailScope::Node(state.node_id), + "backup::database::after_cut", + ) + .await; assemble(state, cut, replies) } diff --git a/nodedb/src/control/backup/restore/redo_reissue/reissue.rs b/nodedb/src/control/backup/restore/redo_reissue/reissue.rs index a896b27fe..eab133411 100644 --- a/nodedb/src/control/backup/restore/redo_reissue/reissue.rs +++ b/nodedb/src/control/backup/restore/redo_reissue/reissue.rs @@ -53,11 +53,15 @@ pub(in crate::control::backup::restore) async fn reissue_rows_and_edges( stats.records += commit_collection(state, tenant, target.restore_id, collection).await?; } // Fails the re-issue after the rows committed and before any edge. - crate::fail_point_err!("restore::reissue::before_edges", |detail: String| { - crate::Error::Internal { - detail: format!("fail point: {detail}"), + crate::fail_point_err!( + crate::fail_point::FailScope::Node(state.node_id), + "restore::reissue::before_edges", + |detail: String| { + crate::Error::Internal { + detail: format!("fail point: {detail}"), + } } - }); + ); let (versions, edges) = edge_units(state, tenant_id, target, rows.edges).await?; stats.edges = versions; // Both homes of an edge commit in one transaction. Each version is diff --git a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/collection.rs b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/collection.rs index 6cb9ce923..e80b09e54 100644 --- a/nodedb/src/control/catalog_entry/post_apply/async_dispatch/collection.rs +++ b/nodedb/src/control/catalog_entry/post_apply/async_dispatch/collection.rs @@ -105,12 +105,16 @@ pub(crate) async fn reclaim_collection_storage( // 1. Persist to redb (every node has its own catalog). A failure here // leaves no durable retry owner, so it is a `no_retry` failure: the caller // releases its lifecycle guard rather than leaking the drain. - crate::fail_point_err!("collection_reclaim::before_tombstone", |detail: String| { - ReclaimFailure::no_retry(crate::Error::Storage { - engine: "catalog".into(), - detail, - }) - }); + crate::fail_point_err!( + crate::fail_point::FailScope::Node(shared.node_id), + "collection_reclaim::before_tombstone", + |detail: String| { + ReclaimFailure::no_retry(crate::Error::Storage { + engine: "catalog".into(), + detail, + }) + } + ); let catalog = shared.credentials.catalog(); catalog .record_wal_tombstone(database_id, tenant_id, name, purge_lsn) diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/passive_read.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/passive_read.rs index 74368bb1b..2528d6043 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/passive_read.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/passive_read.rs @@ -248,10 +248,12 @@ impl Scheduler { let state = Arc::downgrade(&self.shared); #[cfg(feature = "failpoints")] let gates = gates.clone(); + #[cfg(feature = "failpoints")] + let fail_scope = crate::fail_point::FailScope::Node(self.shared.node_id); tokio::spawn(async move { #[cfg(feature = "failpoints")] for gate in &gates { - crate::control::fail_gate::wait(gate).await; + crate::control::fail_gate::wait(fail_scope, gate).await; } let deadline = tokio::time::Instant::now() + budget; if let Err(error) = propose_calvin_read_result(state, proposal, deadline).await { diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs index 719576029..91f142284 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/redo_propose.rs @@ -142,7 +142,10 @@ impl Scheduler { .get(&txn_id) .and_then(|pending| pending.redo.as_ref()) .is_some_and(|owed| { - crate::control::fail_gate::holds_redo_propose(&owed.payload.collections) + crate::control::fail_gate::holds_redo_propose( + self.shared.node_id, + &owed.payload.collections, + ) }); if held { tracing::info!( diff --git a/nodedb/src/control/cluster/metadata_applier/catalog_ddl.rs b/nodedb/src/control/cluster/metadata_applier/catalog_ddl.rs index be9bfdc24..c71d79712 100644 --- a/nodedb/src/control/cluster/metadata_applier/catalog_ddl.rs +++ b/nodedb/src/control/cluster/metadata_applier/catalog_ddl.rs @@ -89,12 +89,16 @@ impl MetadataCommitApplier { stamped, catalog_entry::CatalogEntry::PutBackupScheduleMark(_) ) { - nodedb_types::fail_point_err!(&backup_mark_fail_point(shared.node_id), |detail| { - crate::Error::Storage { - engine: "catalog".into(), - detail, + nodedb_types::fail_point_err!( + nodedb_types::fail_point::FailScope::Node(shared.node_id), + BACKUP_MARK_FAIL_POINT, + |detail| { + crate::Error::Storage { + engine: "catalog".into(), + detail, + } } - }); + ); } // Descriptor versions (and the constraint_version / @@ -175,8 +179,6 @@ impl MetadataCommitApplier { } } -/// The fail point that holds back node `node_id`'s apply of backup schedule -/// marks. -pub fn backup_mark_fail_point(node_id: u64) -> String { - format!("backup_schedule::mark_apply::node{node_id}") -} +/// The fail point that holds back a node's apply of backup schedule marks. +/// A test arms it for one node. +pub const BACKUP_MARK_FAIL_POINT: &str = "backup_schedule::mark_apply"; diff --git a/nodedb/src/control/cluster/metadata_applier/dispatch.rs b/nodedb/src/control/cluster/metadata_applier/dispatch.rs index aece97cff..fdf8343f5 100644 --- a/nodedb/src/control/cluster/metadata_applier/dispatch.rs +++ b/nodedb/src/control/cluster/metadata_applier/dispatch.rs @@ -315,10 +315,9 @@ impl MetadataCommitApplier { } } -/// The fail point that holds back node `node_id`'s whole metadata apply. -pub fn metadata_apply_hold_point(node_id: u64) -> String { - format!("metadata_apply::hold::node{node_id}") -} +/// The fail point that holds back a node's whole metadata apply. A test arms +/// it for one node. +pub const METADATA_APPLY_HOLD_POINT: &str = "metadata_apply::hold"; #[async_trait::async_trait] impl MetadataApplier for MetadataCommitApplier { diff --git a/nodedb/src/control/cluster/metadata_applier/mod.rs b/nodedb/src/control/cluster/metadata_applier/mod.rs index e5c28f67e..68f75ea0f 100644 --- a/nodedb/src/control/cluster/metadata_applier/mod.rs +++ b/nodedb/src/control/cluster/metadata_applier/mod.rs @@ -62,7 +62,7 @@ mod types; mod wedge; pub use boot_seed::{seed_host_tables, seed_metadata_cache}; -pub use catalog_ddl::backup_mark_fail_point; -pub use dispatch::metadata_apply_hold_point; +pub use catalog_ddl::BACKUP_MARK_FAIL_POINT; +pub use dispatch::METADATA_APPLY_HOLD_POINT; pub use types::{CATALOG_CHANNEL_CAPACITY, CatalogChangeEvent, MetadataCommitApplier}; pub use wedge::{ApplyFailureClass, MetadataApplyWedge, WedgeReport, classify}; diff --git a/nodedb/src/control/cluster/snapshot_install/dispatch.rs b/nodedb/src/control/cluster/snapshot_install/dispatch.rs index 762918c57..829f243ad 100644 --- a/nodedb/src/control/cluster/snapshot_install/dispatch.rs +++ b/nodedb/src/control/cluster/snapshot_install/dispatch.rs @@ -137,13 +137,14 @@ fn installed(resp: Response) -> crate::Result<()> { } } -/// Fail point `snapshot_install::node::core`: core `C` of node `N` -/// reports its share as failed after it installed it. +/// Fail point `snapshot_install::core`, scoped to node `node_id`: core +/// `C` of the node reports its share as failed after it installed it. fn injected_fault(node_id: u64, core_id: usize) -> crate::Result<()> { #[cfg(feature = "failpoints")] - if let Some(detail) = - crate::fail_point::eval_fail(&format!("snapshot_install::node{node_id}::core{core_id}")) - { + if let Some(detail) = crate::fail_point::eval_fail( + crate::fail_point::FailScope::Node(node_id), + &format!("snapshot_install::core{core_id}"), + ) { return Err(crate::Error::DataPlane(ErrorCode::Internal { detail })); } #[cfg(not(feature = "failpoints"))] diff --git a/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs b/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs index 0f4184ea6..bfdec2140 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/calvin_redo.rs @@ -252,13 +252,16 @@ impl CalvinClaim { if !matches!(event, CalvinApplyEvent::RedoApplied { .. }) { return Some(event); } - let Some(gate) = crate::control::fail_gate::holds_redo_applied_push(collections) else { + let Some(gate) = + crate::control::fail_gate::holds_redo_applied_push(state.node_id, collections) + else { return Some(event); }; let Some(inbox) = state.calvin.inboxes.get(self.vshard_id) else { return Some(event); }; let (epoch, position) = (self.epoch, self.position); + let fail_scope = crate::fail_point::FailScope::Node(state.node_id); tracing::info!( vshard_id = self.vshard_id, epoch, @@ -267,7 +270,7 @@ impl CalvinClaim { "calvin: RedoApplied withheld at a fail point" ); tokio::spawn(async move { - crate::control::fail_gate::released(&gate).await; + crate::control::fail_gate::released(fail_scope, &gate).await; inbox.push(epoch, position, event).await; }); None diff --git a/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs b/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs index 1c1a97ec1..335ed0c61 100644 --- a/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs +++ b/nodedb/src/control/distributed_applier/apply_loop/redo_chunk.rs @@ -68,7 +68,10 @@ pub(super) fn prepare_stream_entry<'a>( let applied = ctx.state.redo_chunks.apply_chunk(chunk).await; // The chunk's records are durable here, its entry not yet // concluded. - crate::fail_point!("redo_chunk::after_chunk_durable"); + crate::fail_point!( + crate::fail_point::FailScope::Node(ctx.state.node_id), + "redo_chunk::after_chunk_durable" + ); finished(ctx, pos, stream, applied) })) } @@ -218,7 +221,10 @@ pub(super) fn prepare_chunked_final<'a>( }; // Every chunk is durable and the final entry committed; nothing of the // final entry applied yet. - crate::fail_point!("redo_chunk::before_final_install"); + crate::fail_point!( + crate::fail_point::FailScope::Node(ctx.state.node_id), + "redo_chunk::before_final_install" + ); let assembled = match &taken { Some(open) => open.assemble(chunked.count, chunked.len), None => Err(RedoChunkError::NotOpen { diff --git a/nodedb/src/control/fail_gate.rs b/nodedb/src/control/fail_gate.rs index 6cea112b0..264a84e1a 100644 --- a/nodedb/src/control/fail_gate.rs +++ b/nodedb/src/control/fail_gate.rs @@ -14,19 +14,19 @@ use std::time::Duration; use crate::bridge::envelope::PhysicalPlan; -use crate::fail_point::{FailAction, lookup}; +use crate::fail_point::{FailAction, FailScope, lookup}; use crate::types::Lsn; /// How often a parked task checks for its release file. const GATE_POLL: Duration = Duration::from_millis(20); -/// Park until the file armed for `name` with `wait_file()` exists. No-op -/// when nothing is armed for `name`. +/// Park until the file armed for `name` in `scope` with `wait_file()` +/// exists. No-op when nothing is armed for `name` in `scope`. /// /// On arrival the gate creates `.parked`, so a test can wait until the /// task has reached the gate before it acts. -pub(crate) async fn wait(name: &str) { - let Some(FailAction::WaitForFile(path)) = lookup(name) else { +pub(crate) async fn wait(scope: FailScope, name: &str) { + let Some(FailAction::WaitForFile(path)) = lookup(scope, name) else { return; }; let mut parked = path.clone().into_os_string(); @@ -39,90 +39,102 @@ pub(crate) async fn wait(name: &str) { } } -/// The write funnel's gate: after a logged write's WAL record is appended and -/// before any core holds the request. Named per collection, -/// `funnel::before_dispatch::`, and only a write carrying a WAL -/// LSN reaches it. A committed redo parks on the gate of each collection it -/// writes. A replicated write parks inside its apply entry's enqueue, so -/// every later entry of its Raft group waits for it. Only a write of another -/// group applies while it is parked. +/// The write funnel's gate on node `node_id`: after a logged write's WAL +/// record is appended and before any core holds the request. Named per +/// collection, `funnel::before_dispatch::`, and only a write +/// carrying a WAL LSN reaches it. A committed redo parks on the gate of each +/// collection it writes. A replicated write parks inside its apply entry's +/// enqueue, so every later entry of its Raft group waits for it. Only a write +/// of another group applies while it is parked. /// -/// `funnel::before_dispatch::node::` parks the write only on -/// node `N`. An in-process cluster test shares one fail-point registry across -/// its nodes, so it names the node to hold one replica's apply. +/// A test arms the gate for one node to hold one replica's apply. pub(crate) async fn before_dispatch(node_id: u64, plan: &PhysicalPlan, wal_lsn: Option) { if wal_lsn.is_none() { return; } for collection in plan.named_collections() { - wait(&format!("funnel::before_dispatch::{collection}")).await; - wait(&format!( - "funnel::before_dispatch::node{node_id}::{collection}" - )) + wait( + FailScope::Node(node_id), + &format!("funnel::before_dispatch::{collection}"), + ) .await; } } -/// The Calvin submit gate: after a transaction's collection incarnations are -/// stamped and before the sequencer admits it. Named per collection, -/// `calvin::after_stamp::`. A test parks a transaction here, then -/// changes a collection it names before the transaction reaches a replica. -pub(crate) async fn after_calvin_stamp(tx_class: &nodedb_cluster::calvin::types::TxClass) { +/// The Calvin submit gate of node `node_id`: after a transaction's collection +/// incarnations are stamped and before the sequencer admits it. Named per +/// collection, `calvin::after_stamp::`. A test parks a +/// transaction here, then changes a collection it names before the +/// transaction reaches a replica. +pub(crate) async fn after_calvin_stamp( + node_id: u64, + tx_class: &nodedb_cluster::calvin::types::TxClass, +) { for named in &tx_class.incarnations { - wait(&format!("calvin::after_stamp::{}", named.collection)).await; + wait( + FailScope::Node(node_id), + &format!("calvin::after_stamp::{}", named.collection), + ) + .await; } } -/// The Calvin scheduler's redo gate: after a committed slice's redo is -/// resolved and before the data-group leader proposes it. Named per -/// collection, `calvin::before_redo_propose::`. +/// The Calvin scheduler's redo gate on node `node_id`: after a committed +/// slice's redo is resolved and before the data-group leader proposes it. +/// Named per collection, `calvin::before_redo_propose::`. /// /// The scheduler cannot park on a timer, so the gate answers without waiting: /// `true` while the file armed for one of `collections` is absent. The /// scheduler then keeps the slice unproposed and asks again on its next stall /// tick. On its first hold the gate creates `.parked`, so a test can /// wait until the slice is held. -pub(crate) fn holds_redo_propose(collections: &[String]) -> bool { - collections - .iter() - .any(|collection| holds(&format!("calvin::before_redo_propose::{collection}"))) +pub(crate) fn holds_redo_propose(node_id: u64, collections: &[String]) -> bool { + collections.iter().any(|collection| { + holds( + FailScope::Node(node_id), + &format!("calvin::before_redo_propose::{collection}"), + ) + }) } -/// The Calvin install gate: after a stamped redo's install is durable and -/// before its `RedoApplied` event reaches the vShard's scheduler. Named per -/// collection, `calvin::before_redo_applied_push::`. +/// The Calvin install gate on node `node_id`: after a stamped redo's install +/// is durable and before its `RedoApplied` event reaches the vShard's +/// scheduler. Named per collection, +/// `calvin::before_redo_applied_push::`. /// /// The apply loop cannot park here without holding the entry back. Returns /// the name of the gate that holds one of `collections`: its armed file is /// absent. The caller hands the event to its own task, which waits on -/// [`released`]. On its first hold the gate creates `.parked`. -pub(crate) fn holds_redo_applied_push(collections: &[String]) -> Option { +/// [`released`] in the same scope. On its first hold the gate creates +/// `.parked`. +pub(crate) fn holds_redo_applied_push(node_id: u64, collections: &[String]) -> Option { collections .iter() .map(|collection| format!("calvin::before_redo_applied_push::{collection}")) - .find(|name| holds(name)) + .find(|name| holds(FailScope::Node(node_id), name)) } -/// Resolves once gate `name` stops holding: its armed file exists, or -/// nothing is armed for it. -pub(crate) async fn released(name: &str) { - while armed_and_absent(name).is_some() { +/// Resolves once gate `name` in `scope` stops holding: its armed file +/// exists, or nothing is armed for it. +pub(crate) async fn released(scope: FailScope, name: &str) { + while armed_and_absent(scope, name).is_some() { tokio::time::sleep(GATE_POLL).await; } } -/// The file armed for `name` with `wait_file()`, while it is absent. -fn armed_and_absent(name: &str) -> Option { - let Some(FailAction::WaitForFile(path)) = lookup(name) else { +/// The file armed for `name` in `scope` with `wait_file()`, while it +/// is absent. +fn armed_and_absent(scope: FailScope, name: &str) -> Option { + let Some(FailAction::WaitForFile(path)) = lookup(scope, name) else { return None; }; (!path.exists()).then_some(path) } -/// Whether gate `name` holds now: its armed file is absent. On its first -/// hold it creates `.parked`. -fn holds(name: &str) -> bool { - let Some(path) = armed_and_absent(name) else { +/// Whether gate `name` holds in `scope` now: its armed file is absent. On +/// its first hold it creates `.parked`. +fn holds(scope: FailScope, name: &str) -> bool { + let Some(path) = armed_and_absent(scope, name) else { return false; }; let mut parked = path.into_os_string(); @@ -137,9 +149,9 @@ fn holds(name: &str) -> bool { } /// The committed-message delivery gates of node `node_id`: -/// `publish::before_delivery::node` before a delivery pass sends anything, -/// and `publish::before_cursor_commit::node` after a partition's messages -/// are sent and before its cursor is committed. `point` names the gate. +/// `publish::before_delivery` before a delivery pass sends anything, and +/// `publish::before_cursor_commit` after a partition's messages are sent and +/// before its cursor is committed. `point` names the gate. /// /// Returns `false` when the node shuts down while parked: the pass then stops /// where it is, as a node killed at that point does. @@ -148,13 +160,13 @@ pub(crate) async fn publish_delivery( point: &str, shutdown: tokio::sync::watch::Receiver, ) -> bool { - delivery_gate(&format!("publish::{point}::node{node_id}"), shutdown).await + delivery_gate(node_id, &format!("publish::{point}"), shutdown).await } /// The trigger action lane's firing gates of node `node_id`: -/// `trigger::before_firing::node` before a firing pass fires anything, -/// and `trigger::before_cursor_commit::node` after a partition's actions -/// fired and before its cursor is committed. `point` names the gate. +/// `trigger::before_firing` before a firing pass fires anything, and +/// `trigger::before_cursor_commit` after a partition's actions fired and +/// before its cursor is committed. `point` names the gate. /// /// Returns `false` when the node shuts down while parked. pub(crate) async fn action_firing( @@ -162,27 +174,35 @@ pub(crate) async fn action_firing( point: &str, shutdown: tokio::sync::watch::Receiver, ) -> bool { - delivery_gate(&format!("trigger::{point}::node{node_id}"), shutdown).await + delivery_gate(node_id, &format!("trigger::{point}"), shutdown).await } -/// Park on gate `name` until it is released or the node shuts down. -async fn delivery_gate(name: &str, mut shutdown: tokio::sync::watch::Receiver) -> bool { +/// Park on gate `name` of node `node_id` until it is released or the node +/// shuts down. +async fn delivery_gate( + node_id: u64, + name: &str, + mut shutdown: tokio::sync::watch::Receiver, +) -> bool { if *shutdown.borrow() { return false; } tokio::select! { - () = wait(name) => true, + () = wait(FailScope::Node(node_id), name) => true, _ = shutdown.changed() => false, } } -/// The metadata apply gate of node `node_id`, named by -/// [`crate::control::cluster::metadata_applier::metadata_apply_hold_point`]. +/// The metadata apply gate of node `node_id`, named +/// [`crate::control::cluster::metadata_applier::METADATA_APPLY_HOLD_POINT`]. /// /// The metadata applier runs on the Raft loop and cannot park, so the gate /// answers without waiting: `true` while the file armed for it is absent. /// On its first hold the gate creates `.parked`, so a test can wait /// until the node's apply is held. pub(crate) fn holds_metadata_apply(node_id: u64) -> bool { - holds(&crate::control::cluster::metadata_applier::metadata_apply_hold_point(node_id)) + holds( + FailScope::Node(node_id), + crate::control::cluster::metadata_applier::METADATA_APPLY_HOLD_POINT, + ) } diff --git a/nodedb/src/control/pitr/restore_point/create.rs b/nodedb/src/control/pitr/restore_point/create.rs index e3c7cc772..7bdf73d59 100644 --- a/nodedb/src/control/pitr/restore_point/create.rs +++ b/nodedb/src/control/pitr/restore_point/create.rs @@ -30,7 +30,11 @@ pub async fn create_restore_point( // it: a test orders a metadata entry stamped at or above the watermark // before the point. #[cfg(feature = "failpoints")] - crate::control::fail_gate::wait("restore_point::after_watermark").await; + crate::control::fail_gate::wait( + crate::fail_point::FailScope::Node(state.node_id), + "restore_point::after_watermark", + ) + .await; let created_at_ms = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX)) diff --git a/nodedb/src/control/planner/calvin/submit/local.rs b/nodedb/src/control/planner/calvin/submit/local.rs index 51ffc4e9a..043dac370 100644 --- a/nodedb/src/control/planner/calvin/submit/local.rs +++ b/nodedb/src/control/planner/calvin/submit/local.rs @@ -169,7 +169,7 @@ pub(crate) async fn submit_prepared_and_await( timeout: Duration, ) -> crate::Result> { #[cfg(feature = "failpoints")] - crate::control::fail_gate::after_calvin_stamp(&tx_class).await; + crate::control::fail_gate::after_calvin_stamp(state.node_id, &tx_class).await; let fold = ReplyFold::of_tx_class(&tx_class)?; let inbox = state .sequencer_inbox diff --git a/nodedb/src/control/security/permission_tree/event_handler.rs b/nodedb/src/control/security/permission_tree/event_handler.rs index b6a798c81..50c9444b3 100644 --- a/nodedb/src/control/security/permission_tree/event_handler.rs +++ b/nodedb/src/control/security/permission_tree/event_handler.rs @@ -41,7 +41,11 @@ pub async fn apply_ring_events( // is not acknowledged before the step applies it. The gate sits before // the lock, so a reload can still run. #[cfg(feature = "failpoints")] - crate::control::fail_gate::wait("permission_tree::before_apply").await; + crate::control::fail_gate::wait( + crate::fail_point::FailScope::Any, + "permission_tree::before_apply", + ) + .await; { let mut guard = cache.write().await; for event in events { diff --git a/nodedb/src/control/server/native/session/run.rs b/nodedb/src/control/server/native/session/run.rs index 624a3ee33..8bb0f6758 100644 --- a/nodedb/src/control/server/native/session/run.rs +++ b/nodedb/src/control/server/native/session/run.rs @@ -299,7 +299,10 @@ impl NativeSession { }; // Crash-injection coverage verifies that a panic after a request // mutates transaction state still runs detached connection cleanup. - crate::fail_point!("native_session::after_request"); + crate::fail_point!( + crate::fail_point::FailScope::Node(self.state.node_id), + "native_session::after_request" + ); // Cross-shard reads the request made join the transaction's // read-set. crate::control::server::shared::session::pending_shard_reads::record_pending( diff --git a/nodedb/src/control/server/shared/ddl/neutral/tenant/move_tenant/cutover.rs b/nodedb/src/control/server/shared/ddl/neutral/tenant/move_tenant/cutover.rs index 892202da6..71fadc467 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/tenant/move_tenant/cutover.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/tenant/move_tenant/cutover.rs @@ -100,9 +100,11 @@ pub async fn run( } // Fails the cutover after the re-issue and before the proposal. - crate::fail_point_err!("move_tenant::cutover::before_proposal", |detail: String| { - failed(format!("fail point: {detail}")) - }); + crate::fail_point_err!( + crate::fail_point::FailScope::Node(state.node_id), + "move_tenant::cutover::before_proposal", + |detail: String| failed(format!("fail point: {detail}")) + ); // The arrays are read under the DDL preparation lease, so the commit // rekeys exactly the arrays the catalog holds when it is proposed. Its apply rekeyed the arrays and diff --git a/nodedb/src/control/server/shared/session/commit/single_shard.rs b/nodedb/src/control/server/shared/session/commit/single_shard.rs index 87fa6cee8..e61fe5d36 100644 --- a/nodedb/src/control/server/shared/session/commit/single_shard.rs +++ b/nodedb/src/control/server/shared/session/commit/single_shard.rs @@ -201,7 +201,7 @@ async fn commit_redo( target: RedoTarget, payload: &TransactionRedoPayload, ) -> Result { - if let Err(e) = inject_commit_failure() { + if let Err(e) = inject_commit_failure(state.node_id) { return Err(AbortReason::Dispatch(e)); } // The proposal forwards to the group leader and returns once the redo is @@ -220,11 +220,13 @@ async fn commit_redo( } } -/// The `commit::single_shard_redo_commit` fail point. Compiles to `Ok(())` -/// outside the `failpoints` feature. -fn inject_commit_failure() -> crate::Result<()> { - crate::fail_point_err!("commit::single_shard_redo_commit", |detail| { - crate::Error::Internal { detail } - }); +/// The `commit::single_shard_redo_commit` fail point of node `node_id`. +/// Compiles to `Ok(())` outside the `failpoints` feature. +fn inject_commit_failure(node_id: u64) -> crate::Result<()> { + crate::fail_point_err!( + crate::fail_point::FailScope::Node(node_id), + "commit::single_shard_redo_commit", + |detail| crate::Error::Internal { detail } + ); Ok(()) } diff --git a/nodedb/src/data/executor/core_loop/accessors.rs b/nodedb/src/data/executor/core_loop/accessors.rs index 9a50285c3..0c7ee2892 100644 --- a/nodedb/src/data/executor/core_loop/accessors.rs +++ b/nodedb/src/data/executor/core_loop/accessors.rs @@ -173,6 +173,22 @@ impl CoreLoop { self.quarantine_registry = Some(registry); } + /// Scope this core's fail points to node `node_id`, and the fail points + /// of the event producer it holds. Called once by the server bootstrap + /// before the core recovers. + pub fn set_fail_scope(&mut self, node_id: u64) { + self.fail_scope = nodedb_types::fail_point::FailScope::Node(node_id); + #[cfg(feature = "failpoints")] + if let Some(producer) = self.events.producer.as_mut() { + producer.set_fail_scope(self.fail_scope); + } + } + + /// The scope this core's fail points evaluate in. + pub fn fail_scope(&self) -> nodedb_types::fail_point::FailScope { + self.fail_scope + } + /// Install the shared set of collections some Event Plane consumer /// reads. Called once by the server bootstrap before the core recovers. pub fn set_event_interest(&mut self, interest: Arc) { diff --git a/nodedb/src/data/executor/core_loop/event_emit.rs b/nodedb/src/data/executor/core_loop/event_emit.rs index 47ccea942..e50265e79 100644 --- a/nodedb/src/data/executor/core_loop/event_emit.rs +++ b/nodedb/src/data/executor/core_loop/event_emit.rs @@ -125,8 +125,13 @@ impl CoreLoop { } /// Set the Event Plane producer (called after open, before event loop). + /// The producer's fail points take this core's scope. pub fn set_event_producer(&mut self, producer: crate::event::bus::EventProducer) { self.events.producer = Some(producer); + #[cfg(feature = "failpoints")] + if let Some(producer) = self.events.producer.as_mut() { + producer.set_fail_scope(self.fail_scope); + } } /// Emit a write event carrying any [`crate::event::types::RowId`]. diff --git a/nodedb/src/data/executor/core_loop/open.rs b/nodedb/src/data/executor/core_loop/open.rs index b453fba4c..c5018d34f 100644 --- a/nodedb/src/data/executor/core_loop/open.rs +++ b/nodedb/src/data/executor/core_loop/open.rs @@ -96,6 +96,9 @@ impl CoreLoop { Ok(Self { core_id, + // A core opened without a node fires only actions armed for every + // node. The server bootstrap scopes it with `set_fail_scope`. + fail_scope: nodedb_types::fail_point::FailScope::Any, request_rx, response_tx, task_queue: PriorityQueues::new(), diff --git a/nodedb/src/data/executor/core_loop/state.rs b/nodedb/src/data/executor/core_loop/state.rs index 1d3b37efd..612fb50d3 100644 --- a/nodedb/src/data/executor/core_loop/state.rs +++ b/nodedb/src/data/executor/core_loop/state.rs @@ -38,7 +38,8 @@ use super::priority_queues::PriorityQueues; /// This type is intentionally `!Send` — pinned to a single core. pub struct CoreLoop { pub(in crate::data::executor) core_id: usize, - + /// The node this core runs on: the scope its fail points evaluate in. + pub(in crate::data::executor) fail_scope: nodedb_types::fail_point::FailScope, /// SPSC channel: receives requests from Control Plane. pub(in crate::data::executor) request_rx: Consumer, diff --git a/nodedb/src/data/executor/core_loop/tick.rs b/nodedb/src/data/executor/core_loop/tick.rs index 6a01bacad..69cddf6ee 100644 --- a/nodedb/src/data/executor/core_loop/tick.rs +++ b/nodedb/src/data/executor/core_loop/tick.rs @@ -124,7 +124,10 @@ impl CoreLoop { #[cfg(feature = "failpoints")] if task.wal_lsn().is_some() { for collection in task.plan().named_collections() { - crate::fail_point!(&format!("core::after_apply::{collection}")); + crate::fail_point!( + self.fail_scope, + &format!("core::after_apply::{collection}") + ); } } if let Some(group) = journal { @@ -133,7 +136,10 @@ impl CoreLoop { // is stored and its parts are not journalled yet. #[cfg(feature = "failpoints")] for collection in task.plan().named_collections() { - crate::fail_point!(&format!("core::after_capture::{collection}")); + crate::fail_point!( + self.fail_scope, + &format!("core::after_capture::{collection}") + ); } } task.state = TaskState::Completed; diff --git a/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs b/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs index 96f740035..6738bea4a 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs @@ -92,7 +92,7 @@ impl CoreLoop { self.begin_calvin_plan_staging(task, synthetic_txn_id, *tenant_id, plan, body); self.stage_calvin_plan(task, synthetic_txn_id, *tenant_id, plan, &mut staging)?; // Test-only fault boundary after a potentially-mutating stage. - crate::fail_point!("calvin_static::during_overlay_stage"); + crate::fail_point!(self.fail_scope, "calvin_static::during_overlay_stage"); } Ok::(staging.reply) })); diff --git a/nodedb/src/data/executor/handlers/control/reindex/csr.rs b/nodedb/src/data/executor/handlers/control/reindex/csr.rs index 800fbea57..370258059 100644 --- a/nodedb/src/data/executor/handlers/control/reindex/csr.rs +++ b/nodedb/src/data/executor/handlers/control/reindex/csr.rs @@ -38,11 +38,12 @@ impl CoreLoop { }; let token = seed.token(); let memory = self.graph_memory(target); + let fail_scope = self.fail_scope; let (tx, rx) = mpsc::sync_channel::>(1); let spawned = std::thread::Builder::new() .name(format!("reindex-csr-{}", self.core_id)) .spawn(move || { - hold_build(CSR_BUILD_HOLD); + hold_build(fail_scope, CSR_BUILD_HOLD); // The receiver is gone only when the core shut down. let _ = tx.send(seed.build(memory)); }); diff --git a/nodedb/src/data/executor/handlers/control/reindex/fts.rs b/nodedb/src/data/executor/handlers/control/reindex/fts.rs index 4d7e39a1a..3f24276c5 100644 --- a/nodedb/src/data/executor/handlers/control/reindex/fts.rs +++ b/nodedb/src/data/executor/handlers/control/reindex/fts.rs @@ -33,11 +33,12 @@ impl CoreLoop { return Ok(()); }; let token = ticket.token(); + let fail_scope = self.fail_scope; let (tx, rx) = mpsc::sync_channel::>(1); let spawned = std::thread::Builder::new() .name(format!("reindex-fts-{}", self.core_id)) .spawn(move || { - hold_build(FTS_BUILD_HOLD); + hold_build(fail_scope, FTS_BUILD_HOLD); // The receiver is gone only when the core shut down. let _ = tx.send(ticket.read().map(FtsSnapshot::compact)); }); diff --git a/nodedb/src/data/executor/handlers/control/reindex/hold.rs b/nodedb/src/data/executor/handlers/control/reindex/hold.rs index 4d11035e9..0333b0f8e 100644 --- a/nodedb/src/data/executor/handlers/control/reindex/hold.rs +++ b/nodedb/src/data/executor/handlers/control/reindex/hold.rs @@ -6,25 +6,28 @@ //! The hold runs on the rebuild's own OS thread, never on a core. Without //! the `failpoints` feature it compiles to nothing. +use crate::fail_point::FailScope; + /// Holds a full-text rebuild thread before it reads its snapshot. pub const FTS_BUILD_HOLD: &str = "reindex::fts_build_hold"; /// Holds a CSR rebuild thread before it compacts its snapshot. pub const CSR_BUILD_HOLD: &str = "reindex::csr_build_hold"; -/// Hold the calling rebuild thread at failpoint `name`. +/// Hold the calling rebuild thread at failpoint `name`, evaluated in +/// `scope`, the scope of the core that started the rebuild. /// /// `WaitForFile(path)` parks the thread until `path` exists, for at most /// two minutes. Every other action runs as at any failpoint. -pub(super) fn hold_build(name: &str) { +pub(super) fn hold_build(scope: FailScope, name: &str) { #[cfg(feature = "failpoints")] - hold_armed(name); + hold_armed(scope, name); #[cfg(not(feature = "failpoints"))] - let _ = name; + let _ = (scope, name); } #[cfg(feature = "failpoints")] -fn hold_armed(name: &str) { +fn hold_armed(scope: FailScope, name: &str) { use std::time::{Duration, Instant}; use crate::fail_point::{FailAction, eval, lookup}; @@ -32,14 +35,14 @@ fn hold_armed(name: &str) { const HOLD_LIMIT: Duration = Duration::from_secs(120); const POLL: Duration = Duration::from_millis(5); - match lookup(name) { + match lookup(scope, name) { Some(FailAction::WaitForFile(path)) => { let deadline = Instant::now() + HOLD_LIMIT; while !path.exists() && Instant::now() < deadline { std::thread::sleep(POLL); } } - Some(_) => eval(name), + Some(_) => eval(scope, name), None => {} } } diff --git a/nodedb/src/data/executor/handlers/graph_wcc.rs b/nodedb/src/data/executor/handlers/graph_wcc.rs index 78d285619..48284b3f3 100644 --- a/nodedb/src/data/executor/handlers/graph_wcc.rs +++ b/nodedb/src/data/executor/handlers/graph_wcc.rs @@ -144,9 +144,10 @@ impl CoreLoop { // Fails this one core's round, so a test can check that the all-core // gather fails the query instead of merging the other cores. #[cfg(feature = "failpoints")] - if let Some(detail) = - crate::fail_point::eval_fail(&format!("graph::wcc_superstep::core{}", self.core_id)) - { + if let Some(detail) = crate::fail_point::eval_fail( + self.fail_scope, + &format!("graph::wcc_superstep::core{}", self.core_id), + ) { return self.response_error(task, ErrorCode::Internal { detail }); } diff --git a/nodedb/src/data/executor/wal_replay/kv.rs b/nodedb/src/data/executor/wal_replay/kv.rs index a5c826bc1..d6bf3ab68 100644 --- a/nodedb/src/data/executor/wal_replay/kv.rs +++ b/nodedb/src/data/executor/wal_replay/kv.rs @@ -68,7 +68,7 @@ impl CoreLoop { let record_lsn = record.header.lsn; let tombstones = &tombstones.for_database(database_id); - crate::fail_point!("replay::kv_mid_pass"); + crate::fail_point!(self.fail_scope, "replay::kv_mid_pass"); if is_put { // Absolute-overwrite puts — see `kv_put.rs` for the record diff --git a/nodedb/src/data/executor/wal_replay_all.rs b/nodedb/src/data/executor/wal_replay_all.rs index 505ac6d2b..ac725bf95 100644 --- a/nodedb/src/data/executor/wal_replay_all.rs +++ b/nodedb/src/data/executor/wal_replay_all.rs @@ -85,7 +85,7 @@ impl CoreLoop { self.write_index.record_stamps = super::core_loop::write_index::RecordStamps::from_wal(records); - crate::fail_point!("replay::before_engine_passes"); + crate::fail_point!(self.fail_scope, "replay::before_engine_passes"); // Every engine-bearing record class — standalone (autocommit) records // and the sub-records of committed `TransactionRedo` groups alike — @@ -114,7 +114,7 @@ impl CoreLoop { return Err(e); } - crate::fail_point!("replay::before_sync_hwm_pass"); + crate::fail_point!(self.fail_scope, "replay::before_sync_hwm_pass"); // Reconstruct sync HWM maps from SyncSeqAdvance records so // post-restart deduplication is correct. Fatal on error — diff --git a/nodedb/src/data/runtime/params.rs b/nodedb/src/data/runtime/params.rs index 144ae55ca..a9cf895ee 100644 --- a/nodedb/src/data/runtime/params.rs +++ b/nodedb/src/data/runtime/params.rs @@ -14,6 +14,9 @@ use super::config::CoreCompactionConfig; /// Parameters for [`spawn_core`](super::spawn_core). pub struct SpawnCoreParams<'a> { pub core_id: usize, + /// The node this core runs on. Its fail points evaluate in this node's + /// scope. + pub node_id: u64, pub request_rx: Consumer, pub response_tx: Producer, pub data_dir: &'a Path, diff --git a/nodedb/src/data/runtime/spawn.rs b/nodedb/src/data/runtime/spawn.rs index 8d143e181..b116b2caa 100644 --- a/nodedb/src/data/runtime/spawn.rs +++ b/nodedb/src/data/runtime/spawn.rs @@ -29,6 +29,7 @@ pub fn spawn_core( ) -> std::io::Result<(JoinHandle<()>, EventFdNotifier)> { let SpawnCoreParams { core_id, + node_id, request_rx, response_tx, data_dir, @@ -77,6 +78,7 @@ pub fn spawn_core( array_catalog, ) .expect("failed to open CoreLoop engines"); + core.set_fail_scope(node_id); wire_core_dependencies( &mut core, diff --git a/nodedb/src/event/bus.rs b/nodedb/src/event/bus.rs index a66633712..7579f3c26 100644 --- a/nodedb/src/event/bus.rs +++ b/nodedb/src/event/bus.rs @@ -27,20 +27,6 @@ use super::types::WriteEvent; /// Default ring buffer capacity per core (must be power of two). const DEFAULT_EVENT_BUS_CAPACITY: usize = 65_536; -/// Whether a test armed fail point `event::bus::drop::` for -/// `event`'s collection. The event is then lost as a full ring loses it: its -/// sequence is spent and the Event Plane recovers it from the WAL. -#[cfg(feature = "failpoints")] -fn dropped_by_fail_point(event: &WriteEvent) -> bool { - crate::fail_point::eval_fail(&format!("event::bus::drop::{}", event.collection)).is_some() -} - -/// Without the `failpoints` feature no event is dropped here. -#[cfg(not(feature = "failpoints"))] -fn dropped_by_fail_point(_event: &WriteEvent) -> bool { - false -} - /// The producer half given to a Data Plane core. /// /// `Send` at the trait level, but **logically owned by exactly one @@ -69,6 +55,9 @@ pub struct EventProducer { /// disconnect exactly once per producer instead of spamming /// a warning for every dropped event. disconnect_logged: AtomicBool, + /// The scope of the ring-drop fail point: the owning core's node. + #[cfg(feature = "failpoints")] + fail_scope: crate::fail_point::FailScope, } impl EventProducer { @@ -84,7 +73,7 @@ impl EventProducer { pub fn emit(&mut self, mut event: WriteEvent) -> bool { self.numbering.stamp(&mut event); let sequence = event.sequence; - let pushed = if dropped_by_fail_point(&event) { + let pushed = if self.dropped_by_fail_point(&event) { false } else { self.push(event) @@ -95,6 +84,31 @@ impl EventProducer { pushed } + /// Scope this producer's ring-drop fail point to the owning core's node. + #[cfg(feature = "failpoints")] + pub fn set_fail_scope(&mut self, scope: crate::fail_point::FailScope) { + self.fail_scope = scope; + } + + /// Whether a test armed fail point `event::bus::drop::` for + /// `event`'s collection on this producer's node. The event is then lost + /// as a full ring loses it: its sequence is spent and the Event Plane + /// recovers it from the WAL. + #[cfg(feature = "failpoints")] + fn dropped_by_fail_point(&self, event: &WriteEvent) -> bool { + crate::fail_point::eval_fail( + self.fail_scope, + &format!("event::bus::drop::{}", event.collection), + ) + .is_some() + } + + /// Without the `failpoints` feature no event is dropped here. + #[cfg(not(feature = "failpoints"))] + fn dropped_by_fail_point(&self, _event: &WriteEvent) -> bool { + false + } + /// Update the backpressure state, then push `event` onto the ring. fn push(&mut self, event: WriteEvent) -> bool { let util = self.inner.utilization(); @@ -242,6 +256,8 @@ pub fn create_event_bus_with_capacity( progress: Arc::clone(&progress), numbering: RecordNumbering::new(), disconnect_logged: AtomicBool::new(false), + #[cfg(feature = "failpoints")] + fail_scope: crate::fail_point::FailScope::Any, }); consumers.push(EventConsumerRx { diff --git a/nodedb/src/wal/redo/replay.rs b/nodedb/src/wal/redo/replay.rs index 42361c99b..006a92dca 100644 --- a/nodedb/src/wal/redo/replay.rs +++ b/nodedb/src/wal/redo/replay.rs @@ -184,7 +184,7 @@ impl CoreLoop { let ordered = merge_by_lsn(records, &redo_ops); self.replay_vector_wal(&ordered, num_cores, tombstones); - crate::fail_point!("replay::between_engine_passes"); + crate::fail_point!(self.fail_scope(), "replay::between_engine_passes"); self.replay_vector_extended_wal(&ordered, num_cores, tombstones); // Every document row the WAL names, standalone and redo alike, in LSN // order. @@ -203,7 +203,7 @@ impl CoreLoop { // a WAL record is their only durable backing. self.replay_graph_node_label_wal(records, num_cores); - crate::fail_point!("replay::between_standalone_and_redo"); + crate::fail_point!(self.fail_scope(), "replay::between_standalone_and_redo"); // Every graph edge version the WAL names, standalone and redo alike, in // LSN order. The edges' CSR is rebuilt from the `EdgeStore` at open, so diff --git a/nodedb/tests/inproc/cases/event_restart_watermark_catchup.rs b/nodedb/tests/inproc/cases/event_restart_watermark_catchup.rs index 83b179d54..3ebebcdb1 100644 --- a/nodedb/tests/inproc/cases/event_restart_watermark_catchup.rs +++ b/nodedb/tests/inproc/cases/event_restart_watermark_catchup.rs @@ -187,8 +187,9 @@ async fn restart_fires_held_events_the_lane_had_not_fired_once() { let gates = tempfile::tempdir().unwrap(); let release = gates.path().join("before_firing"); let parked = gates.path().join("before_firing.parked"); - let gate = FailGuard::install( - &format!("trigger::before_firing::node{}", server.shared.node_id), + let gate = FailGuard::for_node( + server.shared.node_id, + "trigger::before_firing", FailAction::WaitForFile(release), ); wait_until("the node's firing parks", || parked.exists()).await; From f2c5842821239c2bb4f46bb560f56e7a1f662a22 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sun, 11 Oct 2026 07:24:53 +0800 Subject: [PATCH 18/19] refactor(gateway): move remote streaming dispatch into its own module Split dispatch_remote_stream out of dispatch_remote to keep each file focused on one dispatch mode. --- nodedb/src/control/gateway/dispatch_remote.rs | 231 +---------------- .../control/gateway/dispatch_remote_stream.rs | 237 ++++++++++++++++++ nodedb/src/control/gateway/dispatcher.rs | 3 +- nodedb/src/control/gateway/mod.rs | 1 + 4 files changed, 244 insertions(+), 228 deletions(-) create mode 100644 nodedb/src/control/gateway/dispatch_remote_stream.rs diff --git a/nodedb/src/control/gateway/dispatch_remote.rs b/nodedb/src/control/gateway/dispatch_remote.rs index ec627d6e2..94b6de798 100644 --- a/nodedb/src/control/gateway/dispatch_remote.rs +++ b/nodedb/src/control/gateway/dispatch_remote.rs @@ -2,22 +2,18 @@ //! Remote dispatch: send a plan to a remote node via `ExecuteRequest` RPC. //! -//! Split out of `dispatcher.rs` to keep that file under the project's -//! 500-line limit. Holds the one-shot (`dispatch_remote`) and streaming -//! (`dispatch_remote_stream`) remote dispatch paths used by -//! [`super::dispatcher::dispatch_route`] and -//! [`super::dispatcher::dispatch_route_stream`] respectively. +//! Holds the one-shot path, [`dispatch_remote`], used by +//! [`super::dispatcher::dispatch_route`]. The streaming path lives in +//! [`super::dispatch_remote_stream`]. use std::sync::Arc; -use futures::StreamExt; use nodedb_cluster::ClusterError; use nodedb_cluster::rpc_codec::{ExecuteRequest, RaftRpc}; use tracing::debug; use crate::Error; use crate::control::security::identity::{Permission, required_permission}; -use crate::control::server::result_stream::{ResultStream, RowBatch}; use crate::control::state::SharedState; use crate::types::{DatabaseId, Lsn, TenantId, TraceId, TxnId, VShardId}; use nodedb_physical::physical_plan::wire as plan_wire; @@ -230,231 +226,12 @@ pub(super) async fn dispatch_remote( } } -/// Remote streaming dispatch via the multi-frame `ExecuteStreamRequest` RPC. -/// -/// Returns a [`ResultStream`] that yields the remote shard's row batches as -/// they arrive over QUIC, interleaved by the caller's `select_all` with any -/// local routes. -/// -/// ## Retry-vs-stream split (critical) -/// -/// Leader resolution and the FIRST frame are obtained EAGERLY here: the bidi -/// stream is opened and the first stream item is pulled inside this function. -/// A terminal error that arrives BEFORE any row (`NotLeader`, -/// `DescriptorMismatch`, transport failure on open) is mapped via -/// [`map_typed_cluster_error`] to a retryable [`Error`] and propagated to the -/// gateway's existing not-leader retry loop. Once at least one chunk has been -/// observed, any subsequent error is TERMINAL — it is surfaced as a stream -/// `Err` and never retried (re-running the plan duplicates the rows -/// already streamed to the client). -/// -/// The returned stream re-emits the buffered first batch followed by the rest. -pub(super) async fn dispatch_remote_stream( - args: RemoteDispatchArgs<'_>, -) -> Result { - let RemoteDispatchArgs { - plan, - shared, - node_id, - vshard_id, - tenant_id, - database_id, - trace_id, - deadline_ms, - version_set, - txn_id, - linearizable, - read_groups, - } = args; - let transport = shared.cluster_transport.as_ref().ok_or(Error::Internal { - detail: "gateway: cluster transport not available for remote stream dispatch".into(), - })?; - - // Resolve Exchange nodes before shipping (symmetric with `dispatch_remote`). - // No session-transaction context crosses this boundary yet, so `None`. - let scope = crate::control::server::exchange::ReadScope { - database_id, - tenant_id, - trace_id, - txn_id: None, - linearizable, - }; - let plan = match Box::pin(crate::control::server::exchange::resolve_exchange_in_plan( - shared, plan, scope, - )) - .await? - { - crate::control::server::exchange::Resolved::Plan(p) => *p, - // A streamable child whose Exchange resolved at the coordinator into a - // ready response/stream — re-emit it as a single-batch / forwarded - // stream. These do not occur for the streamable-scan plans routed here, - // but handle exhaustively and behaviour-preservingly. - crate::control::server::exchange::Resolved::Gathered( - resp, - _shard_watermarks, - _shuffle_reads, - ) => { - let batch = RowBatch { - payload: resp.payload.to_vec(), - watermark_lsn: resp.watermark_lsn, - read_versions: resp.read_versions, - }; - return Ok(Box::pin(futures::stream::once(async move { Ok(batch) }))); - } - crate::control::server::exchange::Resolved::Stream(s) => return Ok(s), - }; - - let plan_bytes = plan_wire::encode(&plan).map_err(|e| Error::Internal { - detail: format!("gateway: plan encode failed: {e}"), - })?; - - let descriptor_versions: Vec = version_set - .iter() - .map( - |(name, version)| nodedb_cluster::rpc_codec::DescriptorVersionEntry { - collection: name.clone(), - version: *version, - }, - ) - .collect(); - - let scoped_vshard = scoped_vshard(&plan, vshard_id)?; - let req = RaftRpc::ExecuteStreamRequest(ExecuteRequest { - plan_bytes, - tenant_id: tenant_id.as_u64(), - database_id: database_id.as_u64(), - deadline_remaining_ms: deadline_ms, - trace_id: trace_id.0, - descriptor_versions, - txn_id, - vshard_id: scoped_vshard, - read_groups, - }); - - debug!( - node_id, - vshard_id, - tenant_id = tenant_id.as_u64(), - "gateway: dispatching ExecuteStreamRequest to remote node" - ); - - // Open the stream eagerly. A failure to even open / send the request is a - // pre-row condition: map it like a transport failure in `dispatch_remote` - // so the retry loop routes elsewhere on the next attempt. - let stream = transport - .send_rpc_stream(node_id, req) - .await - .map_err(|e| Error::NotLeader { - vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), - leader_node: 0, - leader_addr: format!("node-{node_id} (stream open error: {e})"), - leader_term: 0, - })?; - // The `async_stream` body is `!Unpin`; pin it on the heap so we can pull - // the eager first frame and then keep the tail around for `.chain`. - let mut stream = Box::pin(stream); - - // Eagerly pull the FIRST frame so a pre-row terminal error is catchable and - // retryable. Any error here is a pre-row error. - let first = match stream.next().await { - Some(Ok((payload, lsn))) => RowBatch { - payload, - watermark_lsn: Lsn::new(lsn), - // The `ExecuteStream` wire chunk carries only the watermark; no - // read version is threaded on this remote path. - read_versions: crate::types::ReadVersions::new(), - }, - Some(Err(e)) => return Err(map_stream_cluster_error(e, vshard_id)), - // Clean EOF with zero rows: a valid empty result. Return an empty stream. - None => return Ok(Box::pin(futures::stream::empty())), - }; - - // Build the result stream: re-emit the buffered first batch, then forward - // the rest. Errors past the first frame are TERMINAL — surfaced as stream - // `Err`, never retried. - let rest = stream.map(move |item| match item { - Ok((payload, lsn)) => Ok(RowBatch { - payload, - watermark_lsn: Lsn::new(lsn), - read_versions: crate::types::ReadVersions::new(), - }), - Err(e) => Err(Error::Dispatch { - detail: format!("remote stream terminal error: {e}"), - }), - }); - - let head = futures::stream::once(async move { Ok(first) }); - Ok(Box::pin(head.chain(rest))) -} - -/// Map a pre-row [`nodedb_cluster::ClusterError`] from a streaming dispatch to -/// an [`Error`]. -/// -/// A typed error (`StreamTerminal`, `ShardExecution`) maps through the same -/// [`map_typed_cluster_error`] used by the one-shot path, so the gateway retry -/// loop handles it identically. A Data-Plane verdict keeps its code. Any other -/// cluster error becomes a transport-style `NotLeader` (leader_node = 0), so -/// the next attempt re-resolves routing rather than re-entrenching an -/// unreachable node. -fn map_stream_cluster_error(err: ClusterError, vshard_id: u64) -> Error { - match err { - ClusterError::StreamTerminal { error, .. } | ClusterError::ShardExecution { error, .. } => { - map_typed_cluster_error(*error, vshard_id) - } - // A verdict from a shard that answered. Retrying it on another route - // repeats it, so it keeps its SQLSTATE. - ClusterError::DataPlane { code } => Error::DataPlane(code.into()), - other @ (ClusterError::Raft(_) - | ClusterError::VShardNotMapped { .. } - | ClusterError::GroupNotFound { .. } - | ClusterError::LearnerNotCaughtUp { .. } - | ClusterError::MigrationInProgress { .. } - | ClusterError::MigrationPauseBudgetExceeded { .. } - | ClusterError::NodeUnreachable { .. } - | ClusterError::GhostNotFound { .. } - | ClusterError::Transport { .. } - | ClusterError::ShardTimeout { .. } - | ClusterError::Unanswered { .. } - | ClusterError::Storage { .. } - | ClusterError::Codec { .. } - | ClusterError::UnsupportedWireVersion { .. } - | ClusterError::CircuitOpen { .. } - | ClusterError::JoinGroupDisappeared { .. } - | ClusterError::JoinCommitTimeout { .. } - | ClusterError::ReadIndexNotLeader { .. } - | ClusterError::ReadIndexTimeout { .. } - | ClusterError::Config { .. } - | ClusterError::MigrationCheckpoint(_) - | ClusterError::MigrationRecovery(_) - | ClusterError::WrongOwner { .. } - | ClusterError::Calvin(_) - | ClusterError::SnapshotCrcMismatch { .. } - | ClusterError::SnapshotOffsetRegression { .. } - | ClusterError::PartialSnapshotCorrupt { .. } - | ClusterError::PartialSnapshotCleanupFailed { .. } - | ClusterError::SnapshotApplyFailed { .. } - | ClusterError::Mirror(_) - | ClusterError::BspBarrier(_) - | ClusterError::VectorGather(_) - | ClusterError::SpatialGather(_) - | ClusterError::Bm25Gather(_) - | ClusterError::TsGather(_) - | ClusterError::ShufflePush(_) - | ClusterError::RemoteUntyped { .. }) => Error::NotLeader { - vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), - leader_node: 0, - leader_addr: format!("stream dispatch error: {other}"), - leader_term: 0, - }, - } -} - /// The vShard a remote receiver must run `plan` on, or `None` when the plan /// fans across the receiver's local cores. /// /// A transaction meta-op names no collection, so only the route's own vShard /// can pick the core that holds the transaction's staging overlay. -fn scoped_vshard( +pub(super) fn scoped_vshard( plan: &nodedb_physical::physical_plan::PhysicalPlan, vshard_id: u64, ) -> Result, Error> { diff --git a/nodedb/src/control/gateway/dispatch_remote_stream.rs b/nodedb/src/control/gateway/dispatch_remote_stream.rs new file mode 100644 index 000000000..3dd04f63a --- /dev/null +++ b/nodedb/src/control/gateway/dispatch_remote_stream.rs @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Remote streaming dispatch: run a plan on a remote node via the +//! multi-frame `ExecuteStreamRequest` RPC. Used by +//! [`super::dispatcher::dispatch_route_stream`]. + +use futures::StreamExt; +use nodedb_cluster::ClusterError; +use nodedb_cluster::rpc_codec::{ExecuteRequest, RaftRpc}; +use nodedb_physical::physical_plan::wire as plan_wire; +use tracing::debug; + +use crate::Error; +use crate::control::server::result_stream::{ResultStream, RowBatch}; +use crate::types::{Lsn, VShardId}; + +use super::cluster_error::map_typed_cluster_error; +use super::dispatch_remote::{RemoteDispatchArgs, scoped_vshard}; + +/// Remote streaming dispatch via the multi-frame `ExecuteStreamRequest` RPC. +/// +/// Returns a [`ResultStream`] that yields the remote shard's row batches as +/// they arrive over QUIC, interleaved by the caller's `select_all` with any +/// local routes. +/// +/// ## Retry-vs-stream split (critical) +/// +/// Leader resolution and the FIRST frame are obtained EAGERLY here: the bidi +/// stream is opened and the first stream item is pulled inside this function. +/// A terminal error that arrives BEFORE any row (`NotLeader`, +/// `DescriptorMismatch`, transport failure on open) is mapped via +/// [`map_typed_cluster_error`] to a retryable [`Error`] and propagated to the +/// gateway's existing not-leader retry loop. Once at least one chunk has been +/// observed, any subsequent error is TERMINAL — it is surfaced as a stream +/// `Err` and never retried (re-running the plan duplicates the rows +/// already streamed to the client). +/// +/// The returned stream re-emits the buffered first batch followed by the rest. +pub(super) async fn dispatch_remote_stream( + args: RemoteDispatchArgs<'_>, +) -> Result { + let RemoteDispatchArgs { + plan, + shared, + node_id, + vshard_id, + tenant_id, + database_id, + trace_id, + deadline_ms, + version_set, + txn_id, + linearizable, + read_groups, + } = args; + let transport = shared.cluster_transport.as_ref().ok_or(Error::Internal { + detail: "gateway: cluster transport not available for remote stream dispatch".into(), + })?; + + // Resolve Exchange nodes before shipping (symmetric with `dispatch_remote`). + // No session-transaction context crosses this boundary yet, so `None`. + let scope = crate::control::server::exchange::ReadScope { + database_id, + tenant_id, + trace_id, + txn_id: None, + linearizable, + }; + let plan = match Box::pin(crate::control::server::exchange::resolve_exchange_in_plan( + shared, plan, scope, + )) + .await? + { + crate::control::server::exchange::Resolved::Plan(p) => *p, + // A streamable child whose Exchange resolved at the coordinator into a + // ready response/stream — re-emit it as a single-batch / forwarded + // stream. These do not occur for the streamable-scan plans routed here, + // but handle exhaustively and behaviour-preservingly. + crate::control::server::exchange::Resolved::Gathered( + resp, + _shard_watermarks, + _shuffle_reads, + ) => { + let batch = RowBatch { + payload: resp.payload.to_vec(), + watermark_lsn: resp.watermark_lsn, + read_versions: resp.read_versions, + }; + return Ok(Box::pin(futures::stream::once(async move { Ok(batch) }))); + } + crate::control::server::exchange::Resolved::Stream(s) => return Ok(s), + }; + + let plan_bytes = plan_wire::encode(&plan).map_err(|e| Error::Internal { + detail: format!("gateway: plan encode failed: {e}"), + })?; + + let descriptor_versions: Vec = version_set + .iter() + .map( + |(name, version)| nodedb_cluster::rpc_codec::DescriptorVersionEntry { + collection: name.clone(), + version: *version, + }, + ) + .collect(); + + let scoped_vshard = scoped_vshard(&plan, vshard_id)?; + let req = RaftRpc::ExecuteStreamRequest(ExecuteRequest { + plan_bytes, + tenant_id: tenant_id.as_u64(), + database_id: database_id.as_u64(), + deadline_remaining_ms: deadline_ms, + trace_id: trace_id.0, + descriptor_versions, + txn_id, + vshard_id: scoped_vshard, + read_groups, + }); + + debug!( + node_id, + vshard_id, + tenant_id = tenant_id.as_u64(), + "gateway: dispatching ExecuteStreamRequest to remote node" + ); + + // Open the stream eagerly. A failure to even open / send the request is a + // pre-row condition: map it like a transport failure in `dispatch_remote` + // so the retry loop routes elsewhere on the next attempt. + let stream = transport + .send_rpc_stream(node_id, req) + .await + .map_err(|e| Error::NotLeader { + vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), + leader_node: 0, + leader_addr: format!("node-{node_id} (stream open error: {e})"), + leader_term: 0, + })?; + // The `async_stream` body is `!Unpin`; pin it on the heap so we can pull + // the eager first frame and then keep the tail around for `.chain`. + let mut stream = Box::pin(stream); + + // Eagerly pull the FIRST frame so a pre-row terminal error is catchable and + // retryable. Any error here is a pre-row error. + let first = match stream.next().await { + Some(Ok((payload, lsn))) => RowBatch { + payload, + watermark_lsn: Lsn::new(lsn), + // The `ExecuteStream` wire chunk carries only the watermark; no + // read version is threaded on this remote path. + read_versions: crate::types::ReadVersions::new(), + }, + Some(Err(e)) => return Err(map_stream_cluster_error(e, vshard_id)), + // Clean EOF with zero rows: a valid empty result. Return an empty stream. + None => return Ok(Box::pin(futures::stream::empty())), + }; + + // Build the result stream: re-emit the buffered first batch, then forward + // the rest. Errors past the first frame are TERMINAL — surfaced as stream + // `Err`, never retried. + let rest = stream.map(move |item| match item { + Ok((payload, lsn)) => Ok(RowBatch { + payload, + watermark_lsn: Lsn::new(lsn), + read_versions: crate::types::ReadVersions::new(), + }), + Err(e) => Err(Error::Dispatch { + detail: format!("remote stream terminal error: {e}"), + }), + }); + + let head = futures::stream::once(async move { Ok(first) }); + Ok(Box::pin(head.chain(rest))) +} + +/// Map a pre-row [`nodedb_cluster::ClusterError`] from a streaming dispatch to +/// an [`Error`]. +/// +/// A typed error (`StreamTerminal`, `ShardExecution`) maps through the same +/// [`map_typed_cluster_error`] used by the one-shot path, so the gateway retry +/// loop handles it identically. A Data-Plane verdict keeps its code. Any other +/// cluster error becomes a transport-style `NotLeader` (leader_node = 0), so +/// the next attempt re-resolves routing rather than re-entrenching an +/// unreachable node. +fn map_stream_cluster_error(err: ClusterError, vshard_id: u64) -> Error { + match err { + ClusterError::StreamTerminal { error, .. } | ClusterError::ShardExecution { error, .. } => { + map_typed_cluster_error(*error, vshard_id) + } + // A verdict from a shard that answered. Retrying it on another route + // repeats it, so it keeps its SQLSTATE. + ClusterError::DataPlane { code } => Error::DataPlane(code.into()), + other @ (ClusterError::Raft(_) + | ClusterError::VShardNotMapped { .. } + | ClusterError::GroupNotFound { .. } + | ClusterError::LearnerNotCaughtUp { .. } + | ClusterError::MigrationInProgress { .. } + | ClusterError::MigrationPauseBudgetExceeded { .. } + | ClusterError::NodeUnreachable { .. } + | ClusterError::GhostNotFound { .. } + | ClusterError::Transport { .. } + | ClusterError::ShardTimeout { .. } + | ClusterError::Unanswered { .. } + | ClusterError::Storage { .. } + | ClusterError::Codec { .. } + | ClusterError::UnsupportedWireVersion { .. } + | ClusterError::CircuitOpen { .. } + | ClusterError::JoinGroupDisappeared { .. } + | ClusterError::JoinCommitTimeout { .. } + | ClusterError::ReadIndexNotLeader { .. } + | ClusterError::ReadIndexTimeout { .. } + | ClusterError::Config { .. } + | ClusterError::MigrationCheckpoint(_) + | ClusterError::MigrationRecovery(_) + | ClusterError::WrongOwner { .. } + | ClusterError::Calvin(_) + | ClusterError::SnapshotCrcMismatch { .. } + | ClusterError::SnapshotOffsetRegression { .. } + | ClusterError::PartialSnapshotCorrupt { .. } + | ClusterError::PartialSnapshotCleanupFailed { .. } + | ClusterError::SnapshotApplyFailed { .. } + | ClusterError::Mirror(_) + | ClusterError::BspBarrier(_) + | ClusterError::VectorGather(_) + | ClusterError::SpatialGather(_) + | ClusterError::Bm25Gather(_) + | ClusterError::TsGather(_) + | ClusterError::ShufflePush(_) + | ClusterError::RemoteUntyped { .. }) => Error::NotLeader { + vshard_id: VShardId::new((vshard_id % VShardId::COUNT as u64) as u32), + leader_node: 0, + leader_addr: format!("stream dispatch error: {other}"), + leader_term: 0, + }, + } +} diff --git a/nodedb/src/control/gateway/dispatcher.rs b/nodedb/src/control/gateway/dispatcher.rs index fecc604f3..79e396855 100644 --- a/nodedb/src/control/gateway/dispatcher.rs +++ b/nodedb/src/control/gateway/dispatcher.rs @@ -16,7 +16,8 @@ use crate::control::state::SharedState; use crate::types::{DatabaseId, Lsn, TenantId, TraceId, TxnId, VShardId}; use super::dispatch_local::{LocalContext, dispatch_local}; -use super::dispatch_remote::{RemoteDispatchArgs, dispatch_remote, dispatch_remote_stream}; +use super::dispatch_remote::{RemoteDispatchArgs, dispatch_remote}; +use super::dispatch_remote_stream::dispatch_remote_stream; use super::read_leg::{confirm_local_read, linearizable_read_groups}; use super::route::{RouteDecision, TaskRoute}; use super::version_check::check_local_descriptor_versions; diff --git a/nodedb/src/control/gateway/mod.rs b/nodedb/src/control/gateway/mod.rs index 873416326..eeecaef3b 100644 --- a/nodedb/src/control/gateway/mod.rs +++ b/nodedb/src/control/gateway/mod.rs @@ -6,6 +6,7 @@ pub mod colocation_guard; pub mod core; pub mod dispatch_local; pub mod dispatch_remote; +pub mod dispatch_remote_stream; pub mod dispatcher; pub mod error_map; pub mod fuser; From 42da7f6e6e1338e9d87889c1f454b075fcf601c6 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Sun, 11 Oct 2026 07:50:46 +0800 Subject: [PATCH 19/19] chore(calvin): annotate wall-clock uses in the scheduler driver Mark each time-based call in the read-result proposal and run loop as liveness-only, with the reason the log order still decides the result. --- .../cluster/calvin/scheduler/driver/core/passive_read.rs | 1 + .../src/control/cluster/calvin/scheduler/driver/core/propose.rs | 2 ++ .../control/cluster/calvin/scheduler/driver/core/run_loop.rs | 2 ++ 3 files changed, 5 insertions(+) diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/passive_read.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/passive_read.rs index 2528d6043..34d2638de 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/passive_read.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/passive_read.rs @@ -255,6 +255,7 @@ impl Scheduler { for gate in &gates { crate::control::fail_gate::wait(fail_scope, gate).await; } + // no-determinism: the proposal's retry budget is liveness only; the log order decides the result. let deadline = tokio::time::Instant::now() + budget; if let Err(error) = propose_calvin_read_result(state, proposal, deadline).await { tracing::warn!( diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/propose.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/propose.rs index 70d7fd42c..be9176007 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/propose.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/propose.rs @@ -104,6 +104,7 @@ pub async fn propose_calvin_read_result( return Ok(()); } Err(error) => { + // no-determinism: the retry budget is liveness only; the log order decides the result. if tokio::time::Instant::now() >= deadline { return Err(error); } @@ -116,6 +117,7 @@ pub async fn propose_calvin_read_result( ); tokio::time::sleep( RETRY_PAUSE + // no-determinism: a retry pause only. .min(deadline.saturating_duration_since(tokio::time::Instant::now())), ) .await; diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs index f4fa916db..8f4142d2b 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/core/run_loop.rs @@ -149,6 +149,7 @@ impl Scheduler { } _ = tokio::time::sleep_until( + // no-determinism: a wake timer only; the restage reads replicated state. restage_at.unwrap_or_else(tokio::time::Instant::now) ), if restage_at.is_some() => { // The next loop pass stages again every txn whose @@ -156,6 +157,7 @@ impl Scheduler { } _ = tokio::time::sleep_until( + // no-determinism: a wake timer only; the timeout entry's log position decides. read_timeout_at.unwrap_or_else(tokio::time::Instant::now) ), if read_timeout_at.is_some() => { // The next loop pass proposes the timeout entry of every