From d86008047bc0bf134eaeb3a339e52dcb25811660 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Tue, 20 Oct 2026 07:37:07 +0800 Subject: [PATCH] feat(txn): plan predicate writes in transactions and read bitemporal state Predicate UPDATE and DELETE inside a session transaction now read their rows at the statement as the transaction sees them. The plan carries the matched surrogates and implicit edges, the stage refuses a mismatched prediction, and a Calvin COMMIT stages exactly those rows and checks them again at its turn. Redo records and admission keys name every collection a transaction writes. Bitemporal collections gain system-time index lookups, version-aware index backfill, point delete and truncate, and CONVERT COLLECTION that rewrites live versions in place. Index lookups fall back to a version scan for valid-time and all-versions reads. Array table functions form relations: ARRAY_AGG and ARRAY_ELEMENTWISE rows are selectable, and temporal reads append a horizon summary row. GraphRAG fusion keys legs by graph node name, then surrogate, so a node name never collides with a surrogate. Response translation resolves primary keys and shapes fusion results accordingly. --- .../cases/cluster_array_region_agg.rs | 26 + .../cases/sql_cluster_cross_node_dml.rs | 2 + .../graph_rag_fusion_naming_cross_node.rs | 350 +++++++++++ .../src/physical_plan/document/op.rs | 6 + nodedb-physical/src/physical_plan/mod.rs | 2 + .../src/physical_plan/written_collections.rs | 14 +- nodedb-sql/src/engine_rules/index_lookup.rs | 11 + nodedb-sql/src/planner/array_fn/table_fn.rs | 30 +- nodedb-sql/src/planner/array_fn/zone.rs | 9 +- .../src/planner/bitmap_emit/predicate.rs | 35 +- nodedb-sql/src/planner/select/array_from.rs | 299 ++++++---- nodedb-sql/src/resolver/array_tvf.rs | 84 ++- .../sql_suite/cases/index_lookup_temporal.rs | 106 ++++ nodedb-sql/tests/sql_suite/cases/mod.rs | 1 + nodedb-test-support/src/tx_commit.rs | 3 +- .../control/clone/resolver/rewrite_engine.rs | 2 + .../cluster/calvin/scheduler/driver/types.rs | 5 +- .../control/planner/calvin/dependent_recon.rs | 11 +- nodedb/src/control/planner/calvin/mod.rs | 2 + .../control/planner/calvin/node_delete_txn.rs | 18 +- nodedb/src/control/planner/calvin/preexec.rs | 15 +- .../control/planner/calvin/retry_budget.rs | 56 ++ .../src/control/planner/calvin/retry_loop.rs | 21 +- .../src/control/planner/calvin/txn_predict.rs | 189 ++++++ .../control/planner/rls_injection/document.rs | 1 + .../output_schema/build/schema.rs | 15 +- .../planner/sql_plan_convert/scan/core.rs | 21 + .../planner/sql_plan_convert/scan/join.rs | 3 + .../visitor/arms_scan_read.rs | 3 +- .../src/control/security/audit/log_commit.rs | 70 ++- .../control/security/catalog/surrogate_pk.rs | 60 ++ .../resolve/exchange/post_process_arm.rs | 30 +- .../server/graph_dispatch/rag_fusion/coord.rs | 42 +- .../http/routes/query/materialized/append.rs | 10 +- .../http/routes/query/materialized/shape.rs | 6 +- .../server/http/routes/query/ndjson.rs | 44 +- .../server/http/routes/result_shape.rs | 33 +- .../native/dispatch/direct_orchestrated.rs | 8 +- .../server/native/dispatch/graph_match.rs | 7 +- .../server/native/dispatch/graph_owner.rs | 2 +- .../server/native/dispatch/response.rs | 6 +- .../server/native/dispatch/single_task.rs | 5 +- .../server/native/dispatch/sorted_read_op.rs | 2 +- .../handler/routing/dispatch_loop/run.rs | 24 +- .../handler/routing/dispatch_loop/task.rs | 19 +- .../handler/routing/gateway_dispatch.rs | 55 +- .../pgwire/handler/routing/gateway_fold.rs | 5 +- .../server/response_shape/calvin_fold.rs | 4 +- .../response_shape/compose/materialized.rs | 72 ++- .../server/response_shape/compose/mod.rs | 4 +- .../control/server/response_shape/request.rs | 5 +- .../server/response_translate/dispatch.rs | 234 ++++++-- .../control/server/response_translate/mod.rs | 2 + .../server/response_translate/pk_resolve.rs | 184 ++++++ .../server/response_translate/rag_fusion.rs | 58 ++ .../server/response_translate/text_hybrid.rs | 103 ++-- .../server/response_translate/vector.rs | 186 +++--- .../collection/dml/parse/dispatch_write.rs | 4 +- .../shared/ddl/neutral/convert/driver.rs | 18 +- .../ddl/neutral/graph_ops/rag_fusion.rs | 11 +- .../server/shared/session/expander_stage.rs | 1 + .../src/control/server/shared/session/mod.rs | 1 + .../shared/session/node_delete_stage.rs | 11 +- .../server/shared/session/predicted_stage.rs | 104 ++++ .../control/server/shared/session/read_set.rs | 1 + .../server/shared/session/staging_gate.rs | 39 +- .../server/shared/sql/staging_predicates.rs | 4 - .../server/shared/statement_exec/answer.rs | 18 +- .../server/shared/statement_exec/task_loop.rs | 9 +- .../shared/write_admission/admission_keys.rs | 2 +- .../predicate/txn_buffering/classify.rs | 1 + nodedb/src/control/source_read/query.rs | 3 +- .../wal_replication/transaction_redo/mod.rs | 5 +- .../transaction_redo/payload.rs | 2 +- .../data/executor/core_loop/filter_match.rs | 52 +- .../core_loop/vector_index_rebuild.rs | 24 +- nodedb/src/data/executor/dispatch/document.rs | 2 + .../data/executor/enforcement/chain_guard.rs | 7 +- .../materialized_sum/divergence.rs | 3 +- .../enforcement/materialized_sum/rmw.rs | 30 +- .../data/executor/enforcement/period_lock.rs | 68 ++- .../data/executor/enforcement/statement.rs | 2 +- .../executor/handlers/bulk_dml/admission.rs | 54 +- .../data/executor/handlers/bulk_dml/delete.rs | 153 +++-- .../handlers/bulk_dml/delete_cascade.rs | 12 +- .../data/executor/handlers/bulk_dml/mod.rs | 2 + .../data/executor/handlers/bulk_dml/scan.rs | 541 ++++++++++++++++-- .../executor/handlers/bulk_dml/txn_match.rs | 154 +++++ .../data/executor/handlers/bulk_dml/update.rs | 91 ++- .../handlers/bulk_dml/update_persist.rs | 29 +- .../handlers/bulk_dml/update_project.rs | 39 +- .../handlers/control/calvin/active_passive.rs | 24 +- .../handlers/control/calvin/shared.rs | 62 +- .../handlers/control/calvin/static_stage.rs | 149 ++++- .../handlers/control/calvin/test_commit.rs | 2 +- .../handlers/control/calvin_active_verify.rs | 129 ----- .../handlers/control/calvin_overlay_stage.rs | 41 +- .../control/calvin_overlay_stage_bulk.rs | 119 ++-- .../calvin_overlay_stage_bulk_dispatch.rs | 112 ++++ .../handlers/control/calvin_passive_read.rs | 5 +- .../handlers/control/calvin_reply/images.rs | 9 +- .../handlers/control/calvin_resolve.rs | 48 +- .../executor/handlers/control/crdt_doc.rs | 3 +- .../src/data/executor/handlers/control/mod.rs | 2 +- nodedb/src/data/executor/handlers/convert.rs | 139 +++-- .../executor/handlers/convert_bitemporal.rs | 242 ++++++++ .../document/index_backfill_versions.rs | 125 ++++ .../executor/handlers/document/index_fetch.rs | 69 ++- .../handlers/document/index_maintenance.rs | 64 ++- .../data/executor/handlers/document/mod.rs | 1 + .../executor/handlers/document/read/fetch.rs | 109 ++-- .../executor/handlers/document/read/scan.rs | 37 +- .../handlers/document/resolve/apply.rs | 5 +- .../handlers/document/resolve/bulk.rs | 20 +- .../handlers/document/resolve/context.rs | 26 +- .../handlers/document/resolve/txn_rows.rs | 47 +- nodedb/src/data/executor/handlers/facet.rs | 71 ++- .../src/data/executor/handlers/graph_rag.rs | 71 ++- .../executor/handlers/graph_rag_triple.rs | 19 +- nodedb/src/data/executor/handlers/mod.rs | 3 + .../executor/handlers/point/apply_delete.rs | 69 +-- .../handlers/point/apply_put/enforce.rs | 14 +- .../data/executor/handlers/point/delete.rs | 9 +- .../executor/handlers/point/delete_version.rs | 187 ++++++ .../src/data/executor/handlers/point/get.rs | 9 +- .../src/data/executor/handlers/point/mod.rs | 1 + .../executor/handlers/point/update/exec.rs | 44 +- .../data/executor/handlers/provider_scan.rs | 23 +- nodedb/src/data/executor/handlers/rag_key.rs | 181 ++++++ .../src/data/executor/handlers/recursive.rs | 5 +- .../executor/handlers/spatial/rtree_scan.rs | 3 +- .../src/data/executor/handlers/text_rows.rs | 71 +-- .../src/data/executor/handlers/text_search.rs | 2 +- .../executor/handlers/text_search_scan.rs | 24 +- .../handlers/transaction/overlay/merge.rs | 39 +- .../transaction/redo_apply/test_commit.rs | 2 +- .../transaction/redo_apply/validate.rs | 10 +- .../handlers/transaction/resolve/document.rs | 25 +- .../transaction/resolve/row_changes/keyed.rs | 10 +- .../transaction/stage_write/dispatch.rs | 78 +-- .../handlers/transaction/stage_write/mod.rs | 2 + .../transaction/stage_write/plan_staging.rs | 36 ++ .../stage_write/stage_bulk_delete.rs | 144 +++-- .../stage_write/stage_bulk_dispatch.rs | 114 ++++ .../stage_write/stage_bulk_update.rs | 97 +--- .../stage_write/stage_current_body.rs | 12 +- .../stage_write/stage_kv_predicate.rs | 22 +- .../transaction/write_version_parity_tests.rs | 2 +- nodedb/src/data/executor/handlers/truncate.rs | 356 ++++++------ .../data/executor/handlers/truncate_row.rs | 168 ++++++ .../handlers/update_from_join_collect.rs | 204 +------ .../handlers/update_from_join_write.rs | 19 +- .../executor/handlers/upsert/exec/batch.rs | 1 - .../handlers/upsert/exec/batch_plan.rs | 15 +- .../executor/handlers/upsert/exec/dispatch.rs | 13 +- .../executor/handlers/vector_search_exec.rs | 3 +- .../data/executor/handlers/vector_write.rs | 49 +- .../executor/handlers/verify_hash_chain.rs | 28 +- .../data/executor/response_codec/graph_rag.rs | 109 ++++ .../src/data/executor/response_codec/hits.rs | 37 -- .../src/data/executor/response_codec/mod.rs | 8 +- .../src/data/executor/response_codec/raw.rs | 288 ++++++++-- nodedb/src/data/executor/scan_normalize.rs | 20 +- nodedb/src/diag/context/data_plane.rs | 70 +++ nodedb/src/diag/context/mod.rs | 3 +- nodedb/src/diag/mod.rs | 22 +- nodedb/src/diag/recording/data_plane.rs | 32 ++ nodedb/src/diag/recording/mod.rs | 5 +- nodedb/src/engine/sparse/btree_index.rs | 67 ++- .../src/engine/sparse/btree_versioned/doc.rs | 80 +++ .../engine/sparse/btree_versioned/index.rs | 33 ++ nodedb/src/event/consumer/run.rs | 24 +- .../tests/inproc/cases/array_sql_relation.rs | 33 +- nodedb/tests/wire/cases/array_agg_relation.rs | 158 +++++ .../wire/cases/bitemporal_index_convert.rs | 307 ++++++++++ .../wire/cases/bitemporal_predicate_writes.rs | 255 +++++++++ .../wire/cases/calvin_txn_edge_update.rs | 100 ++++ .../wire/cases/calvin_txn_predicate_writes.rs | 213 +++++++ .../wire/cases/graph_rag_fusion_naming.rs | 378 ++++++++++++ .../materialized_sum_txn_predicate_writes.rs | 77 +++ nodedb/tests/wire/cases/mod.rs | 7 + nodedb/tests/wire/cases/trigger_e2e.rs | 54 +- 182 files changed, 7751 insertions(+), 2590 deletions(-) create mode 100644 nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/graph_rag_fusion_naming_cross_node.rs rename nodedb/src/control/wal_replication/transaction_redo/collections.rs => nodedb-physical/src/physical_plan/written_collections.rs (82%) create mode 100644 nodedb-sql/tests/sql_suite/cases/index_lookup_temporal.rs create mode 100644 nodedb/src/control/planner/calvin/retry_budget.rs create mode 100644 nodedb/src/control/planner/calvin/txn_predict.rs create mode 100644 nodedb/src/control/server/response_translate/pk_resolve.rs create mode 100644 nodedb/src/control/server/response_translate/rag_fusion.rs create mode 100644 nodedb/src/control/server/shared/session/predicted_stage.rs create mode 100644 nodedb/src/data/executor/handlers/bulk_dml/txn_match.rs delete mode 100644 nodedb/src/data/executor/handlers/control/calvin_active_verify.rs create mode 100644 nodedb/src/data/executor/handlers/control/calvin_overlay_stage_bulk_dispatch.rs create mode 100644 nodedb/src/data/executor/handlers/convert_bitemporal.rs create mode 100644 nodedb/src/data/executor/handlers/document/index_backfill_versions.rs create mode 100644 nodedb/src/data/executor/handlers/point/delete_version.rs create mode 100644 nodedb/src/data/executor/handlers/rag_key.rs create mode 100644 nodedb/src/data/executor/handlers/transaction/stage_write/plan_staging.rs create mode 100644 nodedb/src/data/executor/handlers/transaction/stage_write/stage_bulk_dispatch.rs create mode 100644 nodedb/src/data/executor/handlers/truncate_row.rs create mode 100644 nodedb/src/data/executor/response_codec/graph_rag.rs create mode 100644 nodedb/tests/wire/cases/array_agg_relation.rs create mode 100644 nodedb/tests/wire/cases/bitemporal_index_convert.rs create mode 100644 nodedb/tests/wire/cases/bitemporal_predicate_writes.rs create mode 100644 nodedb/tests/wire/cases/calvin_txn_edge_update.rs create mode 100644 nodedb/tests/wire/cases/calvin_txn_predicate_writes.rs create mode 100644 nodedb/tests/wire/cases/graph_rag_fusion_naming.rs create mode 100644 nodedb/tests/wire/cases/materialized_sum_txn_predicate_writes.rs diff --git a/nodedb-cluster-tests/tests/common_suite/cases/cluster_array_region_agg.rs b/nodedb-cluster-tests/tests/common_suite/cases/cluster_array_region_agg.rs index 419f3b6de..7fbb14aa6 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/cluster_array_region_agg.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/cluster_array_region_agg.rs @@ -137,3 +137,29 @@ async fn cluster_array_agg_groups_by_a_float_dim_value() { } cluster.shutdown().await; } + +/// A grouped `ARRAY_AGG` read as a relation: a WHERE, a computed column and +/// an ORDER BY by alias run over the merged groups on every node. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn cluster_array_agg_result_rows_take_a_relational_tail() { + let cluster = spawn_genome().await; + let sql = "SELECT \"group\" AS chr, ROUND(result, 1) AS total \ + FROM ARRAY_AGG('genome', 'qual', 'sum', 'chr') \ + WHERE result > 10 ORDER BY chr DESC"; + for (idx, node) in cluster.nodes.iter().enumerate() { + let rows = node + .query_named_rows(sql) + .await + .unwrap_or_else(|e| panic!("node {idx}: {sql}: {e}")); + let got: Vec<(f64, f64)> = rows + .iter() + .map(|row| (number(&row["chr"]), number(&row["total"]))) + .collect(); + assert_eq!( + got, + vec![(2.0, 600.0), (1.0, 60.0)], + "node {idx}: filtered groups, descending" + ); + } + cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/dml_suite/cases/sql_cluster_cross_node_dml.rs b/nodedb-cluster-tests/tests/dml_suite/cases/sql_cluster_cross_node_dml.rs index d415d499a..e6cad4bb9 100644 --- a/nodedb-cluster-tests/tests/dml_suite/cases/sql_cluster_cross_node_dml.rs +++ b/nodedb-cluster-tests/tests/dml_suite/cases/sql_cluster_cross_node_dml.rs @@ -57,6 +57,8 @@ mod graph_native_walks_support; mod graph_owner_reads_cross_node; #[path = "../../sql_cluster_cross_node_dml_tests/graph_rag_fusion_cross_node.rs"] mod graph_rag_fusion_cross_node; +#[path = "../../sql_cluster_cross_node_dml_tests/graph_rag_fusion_naming_cross_node.rs"] +mod graph_rag_fusion_naming_cross_node; #[path = "../../sql_cluster_cross_node_dml_tests/graph_traverse_cross_node.rs"] mod graph_traverse_cross_node; #[path = "../../sql_cluster_cross_node_dml_tests/graph_traverse_reverse_cross_node.rs"] diff --git a/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/graph_rag_fusion_naming_cross_node.rs b/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/graph_rag_fusion_naming_cross_node.rs new file mode 100644 index 000000000..6150c64cb --- /dev/null +++ b/nodedb-cluster-tests/tests/sql_cluster_cross_node_dml_tests/graph_rag_fusion_naming_cross_node.rs @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Search responses name rows by primary key on a node outside the home group. +//! +//! With a replication factor of 1, one node replicates the collection's home +//! group. Every row is written through a second node, so the third node's +//! catalog binds none of the collection's keys. That node answers each +//! search by asking the home leader for the keys its catalog lacks: +//! +//! - `GRAPH RAG FUSION`, two-source and three-source, over pgwire, native +//! SQL and the native `GraphRagFusion` opcode, on every node; +//! - a body-less vector search (`SELECT id, vector_distance(...)`); +//! - a hybrid search (`rrf_score(vector_distance(...), bm25_score(...))`). + +use std::time::Duration; + +use nodedb_test_support::native_harness::{open_trust_session, send_request}; +use nodedb_types::protocol::{OpCode, ResponseStatus, TextFields}; + +use crate::common::cluster_harness::{TestCluster, wait_for}; + +const COLL: &str = "rag_naming"; +const STRICT: &str = "rag_naming_strict"; +const SUPERUSER: &str = "nodedb"; +/// Every key the collection holds. +const KEYS: [&str; 4] = ["lonely", "wordy", "a", "b"]; + +/// The statements that build the schemaless collection: `lonely` is the +/// vector-nearest row in no edge, `wordy` is the only `entangled` match and +/// far from the query, and `a -hop-> b` is the graph. +fn collection_writes() -> Vec { + vec![ + format!( + "INSERT INTO {COLL} (id, body, embedding) VALUES ('lonely', 'plain prose', \ + ARRAY[1.0, 0.0])" + ), + format!( + "INSERT INTO {COLL} (id, body, embedding) VALUES ('wordy', 'entangled photons', \ + ARRAY[-1.0, 0.0])" + ), + format!("INSERT INTO {COLL} (id, body, embedding) VALUES ('a', 'beta', ARRAY[0.0, 1.0])"), + format!( + "INSERT INTO {COLL} (id, body, embedding) VALUES ('b', 'gamma', ARRAY[-0.5, -0.5])" + ), + format!("GRAPH INSERT EDGE IN '{COLL}' FROM 'a' TO 'b' TYPE 'hop'"), + ] +} + +/// A fusion answer's `node_id`s, parsed from its JSON text cell. +fn fusion_node_ids(cell: &str) -> Vec { + let parsed: serde_json::Value = + sonic_rs::from_str(cell).unwrap_or_else(|e| panic!("fusion cell is not JSON: {e}: {cell}")); + parsed["results"] + .as_array() + .unwrap_or_else(|| panic!("no results array: {cell}")) + .iter() + .map(|r| { + r["node_id"] + .as_str() + .unwrap_or_else(|| panic!("a result carries no node_id: {r}")) + .to_string() + }) + .collect() +} + +/// The first column of every row a pgwire query returns. +async fn first_column(client: &tokio_postgres::Client, sql: &str) -> Vec { + client + .simple_query(sql) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")) + .iter() + .filter_map(|m| match m { + tokio_postgres::SimpleQueryMessage::Row(row) => row.get(0).map(str::to_owned), + _ => None, + }) + .collect() +} + +/// Every name is one of `keys`, and `expected` is among them. +fn assert_named(names: &[String], keys: &[&str], expected: &str, what: &str) { + assert!( + names.iter().any(|n| n == expected), + "{what}: `{expected}` must answer its key; got {names:?}" + ); + for name in names { + assert!( + keys.contains(&name.as_str()), + "{what}: every row answers a primary key, never a decimal; got {names:?}" + ); + } +} + +fn two_source() -> String { + format!( + "GRAPH RAG FUSION ON {COLL} QUERY ARRAY[1.0, 0.0] VECTOR_FIELD 'embedding' \ + VECTOR_TOP_K 2 EXPANSION_DEPTH 1 EDGE_LABEL 'hop' FINAL_TOP_K 10" + ) +} + +fn three_source() -> String { + format!( + "GRAPH RAG FUSION ON {COLL} QUERY ARRAY[1.0, 0.0] VECTOR_FIELD 'embedding' \ + VECTOR_TOP_K 2 BM25 'entangled' ON 'body' EXPANSION_DEPTH 1 EDGE_LABEL 'hop' \ + FINAL_TOP_K 10 RRF_K (60.0, 60.0, 60.0)" + ) +} + +fn opcode_fields() -> TextFields { + TextFields { + collection: Some(COLL.to_string()), + query_vector: Some(vec![1.0, 0.0]), + vector_top_k: Some(2), + edge_labels: Some(vec!["hop".to_string()]), + expansion_depth: Some(1), + final_top_k: Some(10), + vector_k: Some(60.0), + graph_k: Some(60.0), + vector_field: Some("embedding".to_string()), + ..Default::default() + } +} + +/// A three-node RF1 cluster, each data group on exactly one node. +async fn rf1_cluster() -> TestCluster { + let cluster = TestCluster::spawn_three_with_replication_factor(1) + .await + .expect("3-node RF1 cluster"); + let groups: Vec = { + let routing = cluster.nodes[0] + .shared + .cluster_routing + .as_ref() + .expect("cluster_routing") + .read() + .unwrap_or_else(|p| p.into_inner()); + routing + .group_ids() + .into_iter() + .filter(|g| *g != 0) + .collect() + }; + wait_for( + "each data group has exactly one replica", + Duration::from_secs(30), + Duration::from_millis(50), + || { + groups.iter().all(|&g| { + cluster + .nodes + .iter() + .filter(|node| node.replicates_data_group(g)) + .count() + == 1 + }) + }, + ) + .await; + cluster +} + +/// The indexes of the node that writes and the node that only reads: +/// neither replicates `collection`'s home group. +fn writer_and_reader(cluster: &TestCluster, collection: &str) -> (usize, usize) { + let group = cluster.nodes[0] + .group_id_for_collection(collection) + .expect("the collection's data group"); + let outside: Vec = cluster + .nodes + .iter() + .enumerate() + .filter(|(_, node)| !node.replicates_data_group(group)) + .map(|(i, _)| i) + .collect(); + assert_eq!(outside.len(), 2, "two nodes sit outside the home group"); + (outside[0], outside[1]) +} + +/// Run `ddl` through any leader and wait until every node sees it. +async fn create(cluster: &TestCluster, ddl: &[String], collections: usize) { + for statement in ddl { + cluster + .exec_ddl_on_any_leader(statement) + .await + .unwrap_or_else(|e| panic!("{statement}: {e}")); + } + wait_for( + "every node sees the collections", + Duration::from_secs(10), + Duration::from_millis(50), + || { + cluster + .nodes + .iter() + .all(|n| n.cached_collection_count() >= collections) + }, + ) + .await; + cluster + .wait_for_full_apply_convergence(Duration::from_secs(20)) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn hit_in_no_edge_returns_its_primary_key_on_every_node() { + let cluster = rf1_cluster().await; + create( + &cluster, + &[ + format!("CREATE COLLECTION {COLL}"), + format!("CREATE VECTOR INDEX idx_{COLL}_emb ON {COLL} METRIC cosine DIM 2"), + format!("CREATE SEARCH INDEX idx_{COLL}_fts ON {COLL} FIELDS body ANALYZER 'standard'"), + ], + 1, + ) + .await; + let (writer, reader) = writer_and_reader(&cluster, COLL); + for statement in collection_writes() { + cluster.nodes[writer] + .client + .simple_query(&statement) + .await + .unwrap_or_else(|e| panic!("{statement}: {e}")); + } + cluster + .wait_for_full_apply_convergence(Duration::from_secs(20)) + .await; + + for (idx, node) in cluster.nodes.iter().enumerate() { + for (sql, expected) in [(two_source(), "lonely"), (three_source(), "wordy")] { + let cells = first_column(&node.client, &sql).await; + let cell = cells + .first() + .unwrap_or_else(|| panic!("node {idx}: `{sql}` returned no row")); + assert_named( + &fusion_node_ids(cell), + &KEYS, + expected, + &format!("node {idx} pgwire `{sql}`"), + ); + let native = node + .native_client() + .query(&sql) + .await + .unwrap_or_else(|e| panic!("node {idx}: native `{sql}`: {e}")); + let native_cell = native + .rows + .first() + .and_then(|row| row.first()) + .map(|cell| serde_json::Value::from(cell.clone())) + .and_then(|value| value.as_str().map(str::to_owned)) + .unwrap_or_else(|| panic!("node {idx}: native `{sql}` returned no text cell")); + assert_named( + &fusion_node_ids(&native_cell), + &KEYS, + expected, + &format!("node {idx} native `{sql}`"), + ); + } + let mut session = open_trust_session(node.native_port, SUPERUSER).await; + let response = send_request(&mut session, 2, OpCode::GraphRagFusion, opcode_fields()).await; + assert_eq!( + response.status, + ResponseStatus::Ok, + "node {idx}: native fusion opcode: {response:?}" + ); + let answer = format!("{:?}", response.rows); + assert!( + answer.contains("lonely"), + "node {idx}: the opcode answer names `lonely`: {answer}" + ); + } + + // The reader's catalog binds no key of the collection: a body-less + // vector search and a hybrid search still answer primary keys. + let reader_client = &cluster.nodes[reader].client; + let vector_ids = first_column( + reader_client, + &format!( + "SELECT id, vector_distance(embedding, ARRAY[1.0, 0.0]) AS d FROM {COLL} \ + ORDER BY d LIMIT 2" + ), + ) + .await; + assert_named( + &vector_ids, + &KEYS, + "lonely", + "reader body-less vector search", + ); + let hybrid_ids = first_column( + reader_client, + &format!( + "SELECT id, rrf_score(vector_distance(embedding, ARRAY[1.0, 0.0]), \ + bm25_score(body, 'entangled')) AS score FROM {COLL} ORDER BY score DESC LIMIT 5" + ), + ) + .await; + assert_named(&hybrid_ids, &KEYS, "wordy", "reader hybrid search"); + + cluster.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn strict_collection_hit_returns_its_declared_key_on_every_node() { + let cluster = rf1_cluster().await; + create( + &cluster, + &[ + format!( + "CREATE COLLECTION {STRICT} (code TEXT PRIMARY KEY, embedding VECTOR(2)) \ + WITH (engine = 'document_strict')" + ), + format!("CREATE VECTOR INDEX idx_{STRICT} ON {STRICT} (embedding) METRIC cosine DIM 2"), + ], + 1, + ) + .await; + let (writer, _) = writer_and_reader(&cluster, STRICT); + for (code, v) in [("lonely", "[1.0, 0.0]"), ("far", "[-1.0, 0.0]")] { + let statement = format!("INSERT INTO {STRICT} (code, embedding) VALUES ('{code}', '{v}')"); + cluster.nodes[writer] + .client + .simple_query(&statement) + .await + .unwrap_or_else(|e| panic!("{statement}: {e}")); + } + cluster + .wait_for_full_apply_convergence(Duration::from_secs(20)) + .await; + + let sql = format!( + "GRAPH RAG FUSION ON {STRICT} QUERY ARRAY[1.0, 0.0] VECTOR_FIELD 'embedding' \ + VECTOR_TOP_K 2 EXPANSION_DEPTH 1 FINAL_TOP_K 10" + ); + for (idx, node) in cluster.nodes.iter().enumerate() { + let cells = first_column(&node.client, &sql).await; + let cell = cells + .first() + .unwrap_or_else(|| panic!("node {idx}: `{sql}` returned no row")); + assert_named( + &fusion_node_ids(cell), + &["lonely", "far"], + "lonely", + &format!("node {idx} strict fusion"), + ); + } + + cluster.shutdown().await; +} diff --git a/nodedb-physical/src/physical_plan/document/op.rs b/nodedb-physical/src/physical_plan/document/op.rs index a37f3041b..d4c2716ba 100644 --- a/nodedb-physical/src/physical_plan/document/op.rs +++ b/nodedb-physical/src/physical_plan/document/op.rs @@ -321,6 +321,12 @@ pub enum DocumentOp { projection: Vec, limit: usize, offset: usize, + /// `FOR SYSTEM_TIME AS OF ` cutoff on a bitemporal collection. + /// `None` reads the current state. The versioned index answers at the + /// cutoff and each row is read at its version as of the cutoff. The + /// index holds no valid time and no history listing, so a valid-time + /// or all-versions read never plans this variant. + system_as_of_ms: Option, }, /// Drop all secondary index entries for a field. diff --git a/nodedb-physical/src/physical_plan/mod.rs b/nodedb-physical/src/physical_plan/mod.rs index d2bcbe92d..8aea5328f 100644 --- a/nodedb-physical/src/physical_plan/mod.rs +++ b/nodedb-physical/src/physical_plan/mod.rs @@ -39,6 +39,7 @@ pub mod timeseries; pub mod truncate_target; pub mod vector; pub mod wire; +pub mod written_collections; pub use aggregate_functions::{ AggregateParam, GENERIC_AGGREGATE_FUNCTIONS, PARAMETERIZED_AGGREGATE_FUNCTIONS, @@ -91,3 +92,4 @@ pub use vector::{ VectorResolvedMutation, VectorWriteTargets, }; pub use wire::{decode, encode}; +pub use written_collections::written_collections; diff --git a/nodedb/src/control/wal_replication/transaction_redo/collections.rs b/nodedb-physical/src/physical_plan/written_collections.rs similarity index 82% rename from nodedb/src/control/wal_replication/transaction_redo/collections.rs rename to nodedb-physical/src/physical_plan/written_collections.rs index 43b33aad5..9991306ad 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/collections.rs +++ b/nodedb-physical/src/physical_plan/written_collections.rs @@ -1,15 +1,15 @@ -// SPDX-License-Identifier: BUSL-1.1 +// SPDX-License-Identifier: Apache-2.0 -//! Every collection a transaction's buffered write plans write. +//! Every collection a set of write plans writes. //! -//! The redo apply gives each one a collection-floor write version at the -//! record's LSN, as the transaction batch did for every written plan. A floor -//! on a collection the plans only read is harmless: it makes OCC validation -//! stricter, never looser. +//! The Control Plane names them on a transaction's redo record and on its +//! admission keys. The Data Plane names them when it opens a transaction's +//! staging for a plan. A collection a plan only reads is harmless here: a +//! floor on it makes OCC validation stricter, never looser. use std::collections::BTreeSet; -use nodedb_physical::physical_plan::{GraphOp, MetaOp, PhysicalPlan, SpatialOp, VectorOp}; +use super::{GraphOp, MetaOp, PhysicalPlan, SpatialOp, VectorOp}; /// The written collections of `plans`, sorted and deduplicated. pub fn written_collections(plans: &[PhysicalPlan]) -> Vec { diff --git a/nodedb-sql/src/engine_rules/index_lookup.rs b/nodedb-sql/src/engine_rules/index_lookup.rs index 937f26337..b788ff130 100644 --- a/nodedb-sql/src/engine_rules/index_lookup.rs +++ b/nodedb-sql/src/engine_rules/index_lookup.rs @@ -27,6 +27,17 @@ pub(crate) fn try_document_index_lookup( if !params.sort_keys.is_empty() || !params.window_functions.is_empty() { return None; } + // The versioned index answers the current state and a system-time + // cutoff. It holds no valid time and lists no history, so a valid-time + // or all-versions read scans the versions instead. + if !matches!(params.temporal.valid_time, crate::temporal::ValidTime::Any) + || matches!( + params.temporal.system_time, + nodedb_types::SystemTimeScope::AllVersions + ) + { + return None; + } // Iterate filters to find the first equality candidate that lines up // with a Ready index. Keep the remaining filters as post-filters. diff --git a/nodedb-sql/src/planner/array_fn/table_fn.rs b/nodedb-sql/src/planner/array_fn/table_fn.rs index 776948e97..940179be2 100644 --- a/nodedb-sql/src/planner/array_fn/table_fn.rs +++ b/nodedb-sql/src/planner/array_fn/table_fn.rs @@ -19,9 +19,12 @@ use crate::types_array::{ArrayBinaryOpAst, ArrayReducerAst, ArraySliceAst}; /// One planned array table-valued function call. /// -/// `Slice` and `Project` read cells: their rows carry the array's dims and -/// the requested attrs, so a SELECT reads them as a relation. `Agg` and -/// `Elementwise` emit rows of their own shape and form no such relation. +/// Each call's rows form a relation a SELECT reads: +/// +/// - `Slice` and `Project` read cells. Their rows carry the array's dims and +/// the requested attrs. +/// - `Agg` rows carry `result`, after `group` when it groups by a dim. +/// - `Elementwise` rows carry the `coords` and `attrs` of one cell. /// /// `ARRAY_PROJECT` is a slice over the whole domain with a required attr /// list, so it plans to the slice read and keeps only its own name. @@ -52,6 +55,17 @@ impl ArrayTableFn { ArrayTableFn::Elementwise(_) => "ARRAY_ELEMENTWISE", } } + + /// The array the call names first: the one array of a slice, project or + /// aggregate, or the left operand of an elementwise op. The relation the + /// call's rows form takes its name and default qualifier from it. + pub fn array_name(&self) -> &str { + match self { + ArrayTableFn::Slice(plan) | ArrayTableFn::Project(plan) => &plan.name, + ArrayTableFn::Agg(plan) => &plan.name, + ArrayTableFn::Elementwise(plan) => &plan.left, + } + } } /// Try to intercept a `SELECT * FROM array_xxx(...)` table-valued @@ -536,7 +550,7 @@ mod tests { #[test] fn agg_scalar() { let p = plan_one("SELECT * FROM ARRAY_AGG('g', 'qual', 'sum')").unwrap(); - match p { + match relation_body(p) { SqlPlan::ArrayAgg(ArrayAggPlan { name, attr, @@ -556,7 +570,7 @@ mod tests { #[test] fn agg_grouped() { let p = plan_one("SELECT * FROM ARRAY_AGG('g', 'qual', 'mean', 'chrom')").unwrap(); - match p { + match relation_body(p) { SqlPlan::ArrayAgg(ArrayAggPlan { reducer, group_by_dim, @@ -581,6 +595,7 @@ mod tests { 'POLYGON((0 1, 100 1, 100 3, 0 3, 0 1))', ARRAY['pos', 'chrom'])", ) .unwrap(); + let p = relation_body(p); let SqlPlan::ArrayAgg(ArrayAggPlan { group_by_dim, slice, @@ -608,6 +623,7 @@ mod tests { ARRAY['pos', 'chrom'])", ) .unwrap(); + let p = relation_body(p); let SqlPlan::ArrayAgg(ArrayAggPlan { group_by_dim, slice, @@ -703,7 +719,7 @@ mod tests { fn elementwise_happy() { let p = plan_one("SELECT * FROM ARRAY_ELEMENTWISE('left', 'right', 'add', 'qual')").unwrap(); - assert!(matches!(p, SqlPlan::ArrayElementwise { .. })); + assert!(matches!(relation_body(p), SqlPlan::ArrayElementwise { .. })); } /// An elementwise op reads both operands at its `AS OF` scope. @@ -720,7 +736,7 @@ mod tests { }; let p = crate::planner::select::plan_query(q, &cat(), &FunctionRegistry::new(), temporal) .unwrap(); - match p { + match relation_body(p) { SqlPlan::ArrayElementwise(ArrayElementwisePlan { temporal: got, .. }) => { assert_eq!(got, temporal); } diff --git a/nodedb-sql/src/planner/array_fn/zone.rs b/nodedb-sql/src/planner/array_fn/zone.rs index 5750b80e7..54157b174 100644 --- a/nodedb-sql/src/planner/array_fn/zone.rs +++ b/nodedb-sql/src/planner/array_fn/zone.rs @@ -251,8 +251,13 @@ mod tests { fn agg_zone(sql: &str) -> ArrayZoneAst { match plan(sql).unwrap_or_else(|e| panic!("{sql}: {e}")) { - SqlPlan::ArrayAgg(ArrayAggPlan { zone, .. }) => zone.expect("zone"), - other => panic!("expected ArrayAgg, got {other:?}"), + SqlPlan::Cte(CtePlan { + mut definitions, .. + }) => match definitions.remove(0).1 { + SqlPlan::ArrayAgg(ArrayAggPlan { zone, .. }) => zone.expect("zone"), + other => panic!("expected ArrayAgg, got {other:?}"), + }, + other => panic!("expected Cte, got {other:?}"), } } diff --git a/nodedb-sql/src/planner/bitmap_emit/predicate.rs b/nodedb-sql/src/planner/bitmap_emit/predicate.rs index d7edb67e3..89a2e3c7d 100644 --- a/nodedb-sql/src/planner/bitmap_emit/predicate.rs +++ b/nodedb-sql/src/planner/bitmap_emit/predicate.rs @@ -16,6 +16,9 @@ //! The hint carries the collection name, index field, and the predicate value(s) //! so the converter layer can build an `IndexedFetch` physical sub-plan. +use nodedb_types::SystemTimeScope; + +use crate::temporal::{TemporalScope, ValidTime}; use crate::types::DocumentIndexLookupPlan; use crate::types::{CompareOp, FilterExpr, SqlPlan, SqlValue}; @@ -33,6 +36,24 @@ pub struct BitmapHint { pub primary_value: SqlValue, /// Additional values for IN-list expansion (empty for equality). pub extra_values: Vec, + /// The side's system-time cutoff, `None` for the current state. The + /// bitmap reads the versioned index at this cutoff, so it holds the rows + /// the side reads. + pub system_as_of_ms: Option, +} + +/// The system-time cutoff a side's bitmap reads at. `None` inside the outer +/// `Option` when the index cannot answer the side's scope: it holds no valid +/// time and lists no history, so such a side takes no pushdown. +fn bitmap_cutoff(temporal: &TemporalScope) -> Option> { + if !matches!(temporal.valid_time, ValidTime::Any) { + return None; + } + match temporal.system_time { + SystemTimeScope::Current => Some(None), + SystemTimeScope::AsOf(ms) => Some(Some(ms)), + SystemTimeScope::AllVersions => None, + } } /// Inspect `plan` and return a `BitmapHint` if the plan qualifies for bitmap @@ -47,20 +68,24 @@ pub fn analyze(plan: &SqlPlan) -> Option { collection, field, value, + temporal, .. }) => Some(BitmapHint { collection: collection.clone(), field: field.clone(), primary_value: value.clone(), extra_values: Vec::new(), + system_as_of_ms: bitmap_cutoff(temporal)?, }), // Plain scan: inspect WHERE filters for indexed-column equality / IN-list. SqlPlan::Scan { collection, filters, + temporal, .. } => { + let system_as_of_ms = bitmap_cutoff(temporal)?; // The `SqlPlan::Scan` does not directly carry `IndexSpec` list — that // lives on `TableInfo` which the engine-rules layer consumed when it // chose *not* to rewrite to `DocumentIndexLookup`. However, when the @@ -75,7 +100,7 @@ pub fn analyze(plan: &SqlPlan) -> Option { // index-backed) is emitted unconditionally. // // Future: if `Scan` carries index metadata, extend this arm. - analyze_scan_filters(collection, filters) + analyze_scan_filters(collection, filters, system_as_of_ms) } _ => None, @@ -90,7 +115,11 @@ pub fn analyze(plan: &SqlPlan) -> Option { /// /// This is called for plain `Scan` nodes whose engine rules could not confirm /// a `Ready` index. Returns `None` to be safe. -fn analyze_scan_filters(collection: &str, filters: &[crate::types::Filter]) -> Option { +fn analyze_scan_filters( + collection: &str, + filters: &[crate::types::Filter], + system_as_of_ms: Option, +) -> Option { // Conservative: only emit when there is exactly one filter and it is an // equality or a small IN-list. The caller (engine rules) would have promoted // this to `DocumentIndexLookup` if the index was `Ready`; we skip here to @@ -108,6 +137,7 @@ fn analyze_scan_filters(collection: &str, filters: &[crate::types::Filter]) -> O field: field.clone(), primary_value: value.clone(), extra_values: Vec::new(), + system_as_of_ms, }), FilterExpr::InList { field, values } if !values.is_empty() && values.len() <= MAX_IN_LIST => @@ -119,6 +149,7 @@ fn analyze_scan_filters(collection: &str, filters: &[crate::types::Filter]) -> O field: field.clone(), primary_value: primary, extra_values: iter.collect(), + system_as_of_ms, }) } _ => None, diff --git a/nodedb-sql/src/planner/select/array_from.rs b/nodedb-sql/src/planner/select/array_from.rs index d0b105a91..80748ff00 100644 --- a/nodedb-sql/src/planner/select/array_from.rs +++ b/nodedb-sql/src/planner/select/array_from.rs @@ -2,30 +2,30 @@ //! `FROM ARRAY_*(...)`: an array table-function call as a SELECT's source. //! -//! `ARRAY_SLICE` and `ARRAY_PROJECT` read cells. Their rows form a relation -//! whose columns are the array's dims and the requested attrs, so the SELECT -//! plans over it like over a derived table: the call becomes the one -//! definition of a CTE, and the SELECT reads that definition by name through -//! the generic relational tail (WHERE, projection, GROUP BY, HAVING, ORDER -//! BY, DISTINCT, LIMIT / OFFSET, windows). +//! Every call's rows form a relation, so the SELECT plans over it like over +//! a derived table: the call becomes the one definition of a CTE, and the +//! SELECT reads that definition by name through the generic relational tail +//! (WHERE, projection, GROUP BY, HAVING, ORDER BY, DISTINCT, LIMIT / OFFSET, +//! windows). //! -//! `ARRAY_AGG` and `ARRAY_ELEMENTWISE` emit rows of their own shape. A -//! SELECT over them takes `*` or plain output columns and a row bound; -//! every other clause is refused by name. +//! - `ARRAY_SLICE` and `ARRAY_PROJECT` read cells. Their columns are the +//! array's dims and the requested attrs. +//! - `ARRAY_AGG` and `ARRAY_ELEMENTWISE` emit rows of their own shape. Their +//! columns are the call's output columns. use sqlparser::ast::{self, Select}; use super::array_cell_geom::rewrite_cell_geometry; use super::cte_catalog::CteCatalog; -use super::helpers::{convert_projection, plain_table_factor}; +use super::helpers::plain_table_factor; use super::query_tail::QueryTail; -use super::select_stmt::{PlannedSelect, has_aggregation, plan_select}; +use super::select_stmt::{PlannedSelect, plan_select}; use crate::error::{Result, SqlError}; use crate::functions::registry::FunctionRegistry; use crate::planner::array_fn::{ArrayTableFn, push_down_into_slice, try_plan_array_table_fn}; -use crate::planner::ast_helpers::{single_table_qualifiers, strip_single_table_qualifiers}; -use crate::resolver::array_tvf::{ARRAY_SYSTEM_TIME_COLUMN, array_slice_columns}; -use crate::resolver::columns::TableScope; +use crate::resolver::array_tvf::{ + ARRAY_SYSTEM_TIME_COLUMN, array_slice_columns, array_table_fn_columns, +}; use crate::temporal::TemporalScope; use crate::types::*; @@ -53,7 +53,7 @@ pub(in crate::planner::select) fn try_plan_array_from( ArrayTableFn::Slice(slice) => plan_cell_read(&ctx, "ARRAY_SLICE", slice)?, ArrayTableFn::Project(slice) => plan_cell_read(&ctx, "ARRAY_PROJECT", slice)?, call @ (ArrayTableFn::Agg(_) | ArrayTableFn::Elementwise(_)) => { - plan_non_relational(&ctx, temporal, call)? + plan_result_read(&ctx, call)? } }; Ok(Some(planned)) @@ -73,7 +73,7 @@ fn plan_cell_read( // before the outer SELECT resolves its expressions. let rewritten = rewrite_cell_geometry(ctx.select, ctx.tail.order_by, &view, &read.qualifier)?; let outer = match &rewritten { - None => read.plan_outer(ctx, columns)?, + None => read.plan_outer(ctx, columns, StarRead::Columns)?, Some(rewrite) => { let tail = QueryTail { order_by: rewrite.order_by.as_ref(), @@ -87,13 +87,38 @@ fn plan_cell_read( tail: &tail, statement_output: ctx.statement_output, }; - read.plan_outer(&rewritten_ctx, columns)? + read.plan_outer(&rewritten_ctx, columns, StarRead::Columns)? } }; push_down_into_slice(&mut slice, &view, &outer.plan, &read.name); Ok(read.into_cte(SqlPlan::ArraySlice(slice), outer)) } +/// Plan the SELECT over the rows an `ARRAY_AGG` or `ARRAY_ELEMENTWISE` call +/// emits, as the relation the call names. +/// +/// The relation's columns are the call's output columns. A plain `*` stays a +/// star, so it reads every key the call's rows carry, as the call announces +/// them. +fn plan_result_read(ctx: &OuterContext<'_>, call: ArrayTableFn) -> Result { + let read = RelationRead::new(ctx, call.sql_name(), call.array_name())?; + let columns = array_table_fn_columns(ctx.catalog, &call)?; + let outer = read.plan_outer(ctx, columns, StarRead::SourceRows)?; + Ok(read.into_cte(call.into_plan(), outer)) +} + +/// How a plain `*` in the SELECT list reads the relation. +#[derive(Clone, Copy)] +enum StarRead { + /// One typed column per relation column, in schema order. A cell read + /// announces its dims and attrs this way. + Columns, + /// The star itself. It reads every key the source rows carry, under the + /// names the source announces, so a temporal call keeps its summary-row + /// flag. + SourceRows, +} + /// The parts of the enclosing SELECT every arm reads. struct OuterContext<'a> { select: &'a Select, @@ -146,12 +171,18 @@ impl RelationRead { &self, ctx: &OuterContext<'_>, columns: Vec, + star: StarRead, ) -> Result { - let star_columns: Vec = columns - .iter() - .map(|c| c.name.clone()) - .filter(|name| name != ARRAY_SYSTEM_TIME_COLUMN) - .collect(); + let star_columns: Option> = match star { + StarRead::Columns => Some( + columns + .iter() + .map(|c| c.name.clone()) + .filter(|name| name != ARRAY_SYSTEM_TIME_COLUMN) + .collect(), + ), + StarRead::SourceRows => None, + }; let info = CollectionInfo { name: self.name.clone(), engine: EngineType::DocumentSchemaless, @@ -180,10 +211,12 @@ impl RelationRead { at: None, }), ); - outer_select.projection = - expand_wildcards(&ctx.select.projection, &qualifier, &star_columns)?; + if let Some(star_columns) = &star_columns { + outer_select.projection = + expand_wildcards(&ctx.select.projection, &qualifier, star_columns)?; + } // The temporal qualifier reads the array's versions and rides on the - // slice plan. The relation the SELECT reads is those versions' + // call's plan. The relation the SELECT reads is those versions' // rows, with no history of its own. plan_select( &outer_select, @@ -284,91 +317,6 @@ fn is_plain_wildcard(options: &ast::WildcardAdditionalOptions) -> bool { && options.opt_alias.is_none() } -/// Plan a SELECT over `ARRAY_AGG` / `ARRAY_ELEMENTWISE`. -/// -/// The call's rows are its result. The SELECT may list `*` or the call's -/// own output columns, and bound the rows with LIMIT / FETCH. A filter, -/// grouping, ordering, dedup, offset, window or computed column has no -/// dims-and-attrs relation to read, so each is refused by name. -fn plan_non_relational( - ctx: &OuterContext<'_>, - temporal: TemporalScope, - call: ArrayTableFn, -) -> Result { - let fn_name = call.sql_name(); - let select = ctx.select; - let refuse = |clause: &str| -> Result { - Err(SqlError::Unsupported { - detail: format!( - "{clause} over {fn_name} is not supported: {fn_name} emits its own result \ - rows, not the array's dims and attrs; read the cells with ARRAY_SLICE" - ), - }) - }; - if select.selection.is_some() { - return refuse("WHERE"); - } - if has_aggregation(select, ctx.functions) { - return refuse("GROUP BY or an aggregate"); - } - if select.having.is_some() { - return refuse("HAVING"); - } - if select.qualify.is_some() { - return refuse("QUALIFY"); - } - if !select.named_window.is_empty() { - return refuse("WINDOW"); - } - if select.distinct.is_some() { - return refuse("DISTINCT"); - } - if ctx.tail.order_by.is_some() { - return refuse("ORDER BY"); - } - if ctx.tail.limit_offset()?.1 != 0 { - return refuse("OFFSET"); - } - - let scope = TableScope::resolve_from(ctx.catalog, ctx.functions, temporal, &select.from)?; - let table = scope - .single_table() - .cloned() - .ok_or_else(|| SqlError::Unsupported { - detail: format!("{fn_name}: FROM must name the call alone"), - })?; - let normalized = strip_single_table_qualifiers(select, &single_table_qualifiers(&table))?; - let projection = convert_projection(&normalized.projection, &scope)?; - let plain = projection.iter().all(|p| match p { - Projection::Star | Projection::QualifiedStar(_) | Projection::Column(_) => true, - Projection::Computed { expr, .. } => matches!(expr, SqlExpr::Column { .. }), - Projection::CpComputed { .. } => false, - }); - if !plain { - return refuse("a computed SELECT-list expression"); - } - - let source = call.into_plan(); - let star_only = projection - .iter() - .all(|p| matches!(p, Projection::Star | Projection::QualifiedStar(_))); - let plan = if star_only { - source - } else { - SqlPlan::Subquery { - input: Box::new(source), - filters: Vec::new(), - projection, - window_functions: Vec::new(), - sort_keys: Vec::new(), - offset: 0, - distinct: false, - limit: None, - } - }; - Ok(PlannedSelect { plan, scope }) -} - #[cfg(test)] mod tests { use crate::catalog::{ArrayCatalogView, SqlCatalogError}; @@ -609,33 +557,107 @@ mod tests { } } - #[test] - fn where_over_array_agg_is_refused_by_name() { - let err = plan("SELECT * FROM ARRAY_AGG('img', 'nir', 'sum') WHERE result > 1") - .expect_err("refused"); - let SqlError::Unsupported { detail } = err else { - panic!("expected Unsupported, got {err:?}"); + /// The relation name, the call body and the outer read of a planned + /// `ARRAY_AGG` / `ARRAY_ELEMENTWISE` read. + fn call_parts(plan: SqlPlan) -> (String, SqlPlan, SqlPlan) { + let SqlPlan::Cte(CtePlan { + mut definitions, + outer, + }) = plan + else { + panic!("expected Cte, got {plan:?}"); }; - assert!(detail.starts_with("WHERE over ARRAY_AGG"), "{detail}"); + let (name, body) = definitions.remove(0); + (name, body, *outer) + } + + /// The plan the executor lowers for `sql`: the outer read with the call + /// inlined. + fn inlined(sql: &str) -> SqlPlan { + let (name, body, outer) = call_parts(plan(sql).expect("plans")); + crate::planner::cte::inline_cte(&outer, &name, &body) } #[test] - fn order_by_over_array_elementwise_is_refused_by_name() { - let err = - plan("SELECT * FROM ARRAY_ELEMENTWISE('img', 'img', 'add', 'nir') ORDER BY coords") - .expect_err("refused"); - let SqlError::Unsupported { detail } = err else { - panic!("expected Unsupported, got {err:?}"); + fn order_by_alias_and_round_over_array_agg_ride_the_tail() { + match inlined( + "SELECT \"group\" AS g, ROUND(result, 4) AS r \ + FROM ARRAY_AGG('img', 'nir', 'mean', 'x') ORDER BY g", + ) { + SqlPlan::Subquery { + input, + projection, + sort_keys, + .. + } => { + assert!(matches!(*input, SqlPlan::ArrayAgg(_))); + assert_eq!(sort_keys.len(), 1); + assert!( + projection + .iter() + .any(|p| matches!(p, Projection::Computed { alias, .. } if alias == "r")), + "{projection:?}" + ); + } + other => panic!("expected Subquery over ArrayAgg, got {other:?}"), + } + } + + #[test] + fn order_by_group_with_limit_over_array_agg_rides_the_tail() { + match inlined( + "SELECT \"group\", result FROM ARRAY_AGG('img', 'nir', 'sum', 'x') \ + ORDER BY \"group\" DESC LIMIT 2", + ) { + SqlPlan::Subquery { + input, + sort_keys, + limit, + .. + } => { + assert!(matches!(*input, SqlPlan::ArrayAgg(_))); + assert_eq!(sort_keys.len(), 1); + assert_eq!(limit, Some(2)); + } + other => panic!("expected Subquery over ArrayAgg, got {other:?}"), + } + } + + #[test] + fn where_on_result_over_array_agg_filters_the_rows() { + match inlined("SELECT * FROM ARRAY_AGG('img', 'nir', 'sum') WHERE result > 1") { + SqlPlan::Subquery { input, filters, .. } => { + assert!(matches!(*input, SqlPlan::ArrayAgg(_))); + assert_eq!(filters.len(), 1); + } + other => panic!("expected Subquery over ArrayAgg, got {other:?}"), + } + } + + #[test] + fn star_over_array_agg_reads_the_call_unchanged() { + let (name, body, outer) = + call_parts(plan("SELECT * FROM ARRAY_AGG('img', 'nir', 'sum', 'x')").expect("plans")); + assert_eq!(name, "ARRAY_AGG(img)"); + let SqlPlan::Scan { projection, .. } = &outer else { + panic!("expected outer Scan, got {outer:?}"); }; assert!( - detail.starts_with("ORDER BY over ARRAY_ELEMENTWISE"), - "{detail}" + projection + .iter() + .all(|p| matches!(p, Projection::Star | Projection::QualifiedStar(_))), + "the star stays a star: {projection:?}" ); + assert!(crate::planner::cte::reads_cte_body_unchanged(&outer, &name)); + assert!(matches!( + crate::planner::cte::inline_cte(&outer, &name, &body), + SqlPlan::ArrayAgg(_) + )); } #[test] - fn plain_agg_columns_wrap_in_a_projecting_tail() { - match plan("SELECT result FROM ARRAY_AGG('img', 'nir', 'sum') LIMIT 1").expect("plans") { + fn plain_agg_column_with_limit_rides_the_tail() { + match inlined("SELECT result FROM ARRAY_AGG('img', 'nir', 'sum') LIMIT 1") { SqlPlan::Subquery { input, projection, @@ -649,4 +671,27 @@ mod tests { other => panic!("expected Subquery over ArrayAgg, got {other:?}"), } } + + #[test] + fn group_by_over_array_elementwise_aggregates_the_relation() { + let (name, body, outer) = call_parts( + plan("SELECT COUNT(*) AS n FROM ARRAY_ELEMENTWISE('img', 'img', 'add', 'nir')") + .expect("plans"), + ); + assert_eq!(name, "ARRAY_ELEMENTWISE(img)"); + assert!(matches!(body, SqlPlan::ArrayElementwise(_))); + let SqlPlan::Aggregate { input, .. } = outer else { + panic!("expected Aggregate, got {outer:?}"); + }; + assert!(matches!(*input, SqlPlan::Scan { ref collection, .. } if *collection == name)); + } + + #[test] + fn an_unknown_column_over_array_agg_is_refused() { + let err = plan("SELECT nir FROM ARRAY_AGG('img', 'nir', 'sum')").expect_err("refused"); + assert!( + matches!(err, SqlError::UnknownColumn { .. }), + "unexpected error {err:?}" + ); + } } diff --git a/nodedb-sql/src/resolver/array_tvf.rs b/nodedb-sql/src/resolver/array_tvf.rs index cb55447a3..f62baaee8 100644 --- a/nodedb-sql/src/resolver/array_tvf.rs +++ b/nodedb-sql/src/resolver/array_tvf.rs @@ -156,6 +156,26 @@ pub fn array_agg_columns(view: &ArrayCatalogView, agg: &ArrayAggPlan) -> Result< Ok(cols) } +/// The columns an aggregate's rows carry: [`array_agg_columns`], then +/// [`ARRAY_HORIZON_COLUMN`] when the aggregate is temporal and ends with a +/// summary row. +pub fn array_agg_row_columns( + view: &ArrayCatalogView, + agg: &ArrayAggPlan, +) -> Result> { + let mut cols = array_agg_columns(view, agg)?; + if agg.temporal.is_temporal() { + cols.push(horizon_column()); + } + Ok(cols) +} + +/// The flag column of the summary row a temporal aggregate or elementwise +/// read appends. Every other row holds no value for it. +fn horizon_column() -> ColumnInfo { + column(ARRAY_HORIZON_COLUMN, SqlDataType::Bool, true) +} + /// Refuse a reducer that an attr of type `attr` cannot answer. /// /// The engine folds only numeric cells and skips every other cell. @@ -193,6 +213,8 @@ pub fn check_agg_reducer( /// - A slice or project row holds the array's dims and the requested attrs. /// - An aggregate row holds `result`, after `group` when it groups by a dim. /// - An elementwise row holds the `coords` and `attrs` arrays of one cell. +/// - A temporal aggregate or elementwise read also holds +/// [`ARRAY_HORIZON_COLUMN`], set on its trailing summary row. pub fn array_table_fn_columns( catalog: &dyn SqlCatalog, call: &ArrayTableFn, @@ -201,11 +223,17 @@ pub fn array_table_fn_columns( ArrayTableFn::Slice(slice) | ArrayTableFn::Project(slice) => { Ok(array_slice_columns(&view_of(catalog, &slice.name)?, slice)) } - ArrayTableFn::Agg(agg) => array_agg_columns(&view_of(catalog, &agg.name)?, agg), - ArrayTableFn::Elementwise(_) => Ok(vec![ - column("coords", SqlDataType::Json, false), - column("attrs", SqlDataType::Json, false), - ]), + ArrayTableFn::Agg(agg) => array_agg_row_columns(&view_of(catalog, &agg.name)?, agg), + ArrayTableFn::Elementwise(elementwise) => { + let mut cols = vec![ + column("coords", SqlDataType::Json, false), + column("attrs", SqlDataType::Json, false), + ]; + if elementwise.temporal.is_temporal() { + cols.push(horizon_column()); + } + Ok(cols) + } } } @@ -352,6 +380,52 @@ mod tests { assert_eq!(min.int_width, Some(IntWidth::I16)); } + #[test] + fn temporal_aggregate_rows_carry_the_horizon_flag() { + use crate::types_array::{ArrayAttrAst, ArrayDimAst, ArrayDomainBound, ArraySliceAst}; + let view = ArrayCatalogView { + name: "img".into(), + dims: vec![ArrayDimAst { + name: "x".into(), + dtype: ArrayDimType::Int64, + lo: ArrayDomainBound::Int64(0), + hi: ArrayDomainBound::Int64(3), + }], + attrs: vec![ArrayAttrAst { + name: "nir".into(), + dtype: ArrayAttrType::Float64, + nullable: true, + }], + tile_extents: vec![2], + georef: None, + }; + let agg = |temporal: TemporalScope| ArrayAggPlan { + name: "img".into(), + attr: "nir".into(), + reducer: ArrayReducerAst::Sum, + group_by_dim: Some("x".into()), + slice: ArraySliceAst::default(), + zone: None, + temporal, + }; + let names = |plan: &ArrayAggPlan| -> Vec { + array_agg_row_columns(&view, plan) + .expect("answerable") + .into_iter() + .map(|c| c.name) + .collect() + }; + assert_eq!(names(&agg(TemporalScope::default())), ["group", "result"]); + let as_of = TemporalScope { + system_time: nodedb_types::SystemTimeScope::AsOf(1), + ..TemporalScope::default() + }; + assert_eq!( + names(&agg(as_of)), + ["group", "result", ARRAY_HORIZON_COLUMN] + ); + } + #[test] fn non_numeric_attrs_refuse_numeric_reducers() { for attr in [ArrayAttrType::String, ArrayAttrType::Bytes] { diff --git a/nodedb-sql/tests/sql_suite/cases/index_lookup_temporal.rs b/nodedb-sql/tests/sql_suite/cases/index_lookup_temporal.rs new file mode 100644 index 000000000..379f8c7d7 --- /dev/null +++ b/nodedb-sql/tests/sql_suite/cases/index_lookup_temporal.rs @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! An indexed equality under a time scope. +//! +//! The versioned index answers the current state and a system-time cutoff, +//! so `AS OF SYSTEM TIME ` plans the index lookup and carries the scope +//! to the physical read. The index holds no valid time and lists no history, +//! so a valid-time read and `AS OF SYSTEM TIME NULL` plan a scan. + +use nodedb_sql::types::{ + CollectionInfo, DocumentIndexLookupPlan, EngineType, IndexSpec, IndexState, SqlPlan, +}; +use nodedb_sql::{SqlCatalog, SqlCatalogError, plan_sql}; +use nodedb_types::{DatabaseId, SystemTimeScope}; + +fn text_column(name: &str, is_primary_key: bool) -> nodedb_sql::types::ColumnInfo { + nodedb_sql::types::ColumnInfo { + name: name.into(), + data_type: nodedb_sql::types::SqlDataType::String, + nullable: !is_primary_key, + is_primary_key, + default: None, + raw_type: None, + int_width: None, + float_width: None, + } +} + +/// A bitemporal strict `ledger` with a ready index on `kind`. +struct Catalog; + +impl SqlCatalog for Catalog { + fn get_collection( + &self, + _: DatabaseId, + name: &str, + ) -> std::result::Result, SqlCatalogError> { + Ok((name == "ledger").then(|| CollectionInfo { + name: name.into(), + engine: EngineType::DocumentStrict, + columns: vec![text_column("id", true), text_column("kind", false)], + primary_key: Some("id".into()), + has_auto_tier: false, + indexes: vec![IndexSpec { + name: "ledger_kind".into(), + field: "$.kind".into(), + unique: false, + case_insensitive: false, + state: IndexState::Ready, + predicate: None, + }], + bitemporal: true, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentStrict), + })) + } + + fn lookup_array(&self, _name: &str) -> Option { + None + } + + fn array_exists(&self, _name: &str) -> bool { + false + } +} + +fn plan_one(sql: &str) -> SqlPlan { + let mut plans = plan_sql(sql, &Catalog).expect("planning must succeed"); + assert_eq!(plans.len(), 1, "one plan for {sql}"); + plans.remove(0) +} + +#[test] +fn as_of_indexed_equality_plans_the_index_lookup_with_its_cutoff() { + let plan = plan_one("SELECT id, kind FROM ledger AS OF SYSTEM TIME 1000 WHERE kind = 'x'"); + match plan { + SqlPlan::DocumentIndexLookup(DocumentIndexLookupPlan { + field, temporal, .. + }) => { + assert_eq!(field, "$.kind"); + assert_eq!(temporal.system_time, SystemTimeScope::AsOf(1000)); + } + other => panic!("expected an index lookup, got {other:?}"), + } +} + +#[test] +fn current_indexed_equality_plans_the_index_lookup() { + let plan = plan_one("SELECT id, kind FROM ledger WHERE kind = 'x'"); + assert!( + matches!( + plan, + SqlPlan::DocumentIndexLookup(DocumentIndexLookupPlan { ref temporal, .. }) + if temporal.system_time == SystemTimeScope::Current + ), + "got {plan:?}" + ); +} + +#[test] +fn all_versions_indexed_equality_plans_a_scan() { + let plan = plan_one("SELECT id, kind FROM ledger AS OF SYSTEM TIME NULL WHERE kind = 'x'"); + assert!(matches!(plan, SqlPlan::Scan { .. }), "got {plan:?}"); +} diff --git a/nodedb-sql/tests/sql_suite/cases/mod.rs b/nodedb-sql/tests/sql_suite/cases/mod.rs index f07351615..f42a698e1 100644 --- a/nodedb-sql/tests/sql_suite/cases/mod.rs +++ b/nodedb-sql/tests/sql_suite/cases/mod.rs @@ -2,6 +2,7 @@ mod aggregate_filter_planning; mod ddl_constraint_rejection; +mod index_lookup_temporal; mod limit_offset_bounds; mod on_conflict_update_range_check; mod point_get_operand_order; diff --git a/nodedb-test-support/src/tx_commit.rs b/nodedb-test-support/src/tx_commit.rs index 31cbc7987..a5560a815 100644 --- a/nodedb-test-support/src/tx_commit.rs +++ b/nodedb-test-support/src/tx_commit.rs @@ -4,12 +4,11 @@ use nodedb::bridge::dispatch::{BridgeRequest, BridgeResponse}; use nodedb::bridge::envelope::{Request, Response, Status}; -use nodedb::control::wal_replication::transaction_redo::collections::written_collections; use nodedb::control::wal_replication::transaction_redo::sum_targets::redo_sum_targets; use nodedb::data::executor::core_loop::CoreLoop; use nodedb::types::Lsn; use nodedb_bridge::buffer::{Consumer, Producer}; -use nodedb_physical::physical_plan::{MetaOp, PhysicalPlan}; +use nodedb_physical::physical_plan::{MetaOp, PhysicalPlan, written_collections}; use crate::tx_batch_helpers::make_request; diff --git a/nodedb/src/control/clone/resolver/rewrite_engine.rs b/nodedb/src/control/clone/resolver/rewrite_engine.rs index 7d8224934..7a800be2f 100644 --- a/nodedb/src/control/clone/resolver/rewrite_engine.rs +++ b/nodedb/src/control/clone/resolver/rewrite_engine.rs @@ -97,6 +97,7 @@ pub(super) async fn rewrite_document( projection, limit, offset, + system_as_of_ms, } if ctx.is_target(collection) => Ok(SourceRewrite::task(PhysicalPlan::Document( DocumentOp::IndexedFetch { collection: ctx.source_qualified.clone(), @@ -106,6 +107,7 @@ pub(super) async fn rewrite_document( projection: projection.clone(), limit: *limit, offset: *offset, + system_as_of_ms: *system_as_of_ms, }, ))), diff --git a/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs b/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs index 8de4414e7..396e4419f 100644 --- a/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs +++ b/nodedb/src/control/cluster/calvin/scheduler/driver/types.rs @@ -119,10 +119,7 @@ impl SliceScope { pub(super) fn of_plans(plans: &[nodedb_physical::physical_plan::PhysicalPlan]) -> Self { Self { writes: !plans.is_empty(), - collections: - crate::control::wal_replication::transaction_redo::collections::written_collections( - plans, - ), + collections: nodedb_physical::physical_plan::written_collections(plans), sum_targets: crate::control::wal_replication::transaction_redo::sum_targets::redo_sum_targets( plans, diff --git a/nodedb/src/control/planner/calvin/dependent_recon.rs b/nodedb/src/control/planner/calvin/dependent_recon.rs index c9613c1ff..ade4f953c 100644 --- a/nodedb/src/control/planner/calvin/dependent_recon.rs +++ b/nodedb/src/control/planner/calvin/dependent_recon.rs @@ -297,15 +297,8 @@ async fn dispatch_dependent_edge_recon_inner( // // `predicted_edges` mirrors the recon `edges` (which carry the // surrogate of each edge doc) into the plan-carried wire type. - let predicted_edges: Vec = edges - .iter() - .map(|e| OllpPredictedEdge { - surrogate: e.surrogate, - from: e.from.clone(), - to: e.to.clone(), - label: e.label.clone(), - }) - .collect(); + let predicted_edges: Vec = + edges.iter().map(OllpPredictedEdge::from).collect(); let mut edge_tasks: Vec = Vec::new(); // Node-delete guards run before every other task of the diff --git a/nodedb/src/control/planner/calvin/mod.rs b/nodedb/src/control/planner/calvin/mod.rs index e1018a3d9..efd331f29 100644 --- a/nodedb/src/control/planner/calvin/mod.rs +++ b/nodedb/src/control/planner/calvin/mod.rs @@ -18,9 +18,11 @@ pub mod predicate; pub mod preexec; pub mod read_dependent; pub mod reservation; +pub mod retry_budget; pub mod retry_loop; pub mod submit; pub mod tx_class; +pub mod txn_predict; pub mod types; pub mod write_class; diff --git a/nodedb/src/control/planner/calvin/node_delete_txn.rs b/nodedb/src/control/planner/calvin/node_delete_txn.rs index fbb01d825..6a3639aa3 100644 --- a/nodedb/src/control/planner/calvin/node_delete_txn.rs +++ b/nodedb/src/control/planner/calvin/node_delete_txn.rs @@ -2,8 +2,8 @@ //! Node-delete edge tasks for a delete inside a session transaction. //! -//! An explicit transaction does not take the OLLP reconnaissance route, so -//! the delete's edge tasks are derived at the statement and buffered with it: +//! A transaction derives the delete's edge tasks at the statement, from the +//! read its prediction takes, and buffers them with it: //! //! - One `EdgeDelete` per edge an edge document's `_from`/`_to` mirrors. //! - One `EdgeDelete` per edge of the collection incident on each deleted @@ -19,7 +19,7 @@ use super::dependent_recon_node_edges::{ NodeDeletePlan, NodeIncidentEdges, node_delete_tasks, planned_edge_deletes, read_node_incident_edges, }; -use super::preexec::{PreexecRequest, RowIdentityRead, run_preexec_scan}; +use super::preexec::{PreexecRequest, PreexecScan, RowIdentityRead, run_preexec_scan}; use crate::control::planner::implicit_edges::append_implicit_edge_delete_tasks; use crate::control::state::SharedState; use crate::types::{TraceId, TxnId}; @@ -30,11 +30,15 @@ use crate::types::{TraceId, TxnId}; /// other task, and for a delete on a collection no edge was ever written /// into. A TRUNCATE of an edge-bearing collection is refused. /// +/// `prediction` is the statement's own read of a predicate delete's rows. +/// The edge tasks derive from it, so they cover exactly the rows it stages. +/// /// Guards come first in the returned list. pub async fn txn_node_delete_tasks( state: &SharedState, task: &PhysicalTask, txn_id: Option, + prediction: Option<&PreexecScan>, ) -> crate::Result> { let (tenant_id, database_id) = (task.tenant_id, task.database_id); // The rows the statement deletes: a predicate, one row by surrogate, or @@ -108,8 +112,10 @@ pub async fn txn_node_delete_tasks( let declared_primary_key = coll.declared_primary_key; // The rows as this transaction sees them, read before the delete stages. - let (identities, scanned_edges) = match target { - DeleteTarget::Rows { filters, prefilter } => { + let (identities, scanned_edges) = match (target, prediction) { + // The statement's prediction read these rows already. + (DeleteTarget::Rows { .. }, Some(scan)) => (scan.identities.clone(), scan.edges.clone()), + (DeleteTarget::Rows { filters, prefilter }, None) => { let scan = run_preexec_scan( state, tenant_id, @@ -127,7 +133,7 @@ pub async fn txn_node_delete_tasks( .await?; (scan.identities, scan.edges) } - DeleteTarget::Document(document_id) => (vec![document_id], Vec::new()), + (DeleteTarget::Document(document_id), _) => (vec![document_id], Vec::new()), }; let mut edge_tasks = Vec::new(); append_implicit_edge_delete_tasks( diff --git a/nodedb/src/control/planner/calvin/preexec.rs b/nodedb/src/control/planner/calvin/preexec.rs index 31c7dedb2..24a54c6b4 100644 --- a/nodedb/src/control/planner/calvin/preexec.rs +++ b/nodedb/src/control/planner/calvin/preexec.rs @@ -24,7 +24,7 @@ use nodedb_types::{ use super::dependent_recon_node_edges::NodeIncidentEdges; use crate::control::state::SharedState; use crate::types::{DatabaseId, TraceId, TxnId}; -use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan}; +use nodedb_physical::physical_plan::{DocumentOp, OllpPredictedEdge, PhysicalPlan}; /// One implicit graph edge surfaced from the pre-execution reconnaissance scan. /// @@ -55,6 +55,19 @@ pub struct ScannedEdge { pub weight: Option, } +/// The plan-carried prediction of a scanned edge: the edge document's +/// surrogate and its mirrored `_from`/`_to`/`_type`. +impl From<&ScannedEdge> for OllpPredictedEdge { + fn from(edge: &ScannedEdge) -> Self { + Self { + surrogate: edge.surrogate, + from: edge.from.clone(), + to: edge.to.clone(), + label: edge.label.clone(), + } + } +} + /// Whether the recon scan reads each matched row's identity, and which /// column holds it. #[derive(Clone, Copy)] diff --git a/nodedb/src/control/planner/calvin/retry_budget.rs b/nodedb/src/control/planner/calvin/retry_budget.rs new file mode 100644 index 000000000..09e233545 --- /dev/null +++ b/nodedb/src/control/planner/calvin/retry_budget.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The one OLLP retry policy: a retry budget, and the backoff and circuit +//! accounting the orchestrator applies to each retry. +//! +//! Every Control-Plane loop that re-reads a moved prediction spends retries +//! from this budget: the coordinator's dependent-read loop and a transaction +//! statement's predicate stage. + +use crate::control::cluster::calvin::executor::ollp::orchestrator::OllpOrchestrator; + +/// The retries one OLLP statement may spend on a predicate class. +pub struct OllpRetryBudget<'a> { + orchestrator: &'a OllpOrchestrator, + predicate_class: u64, + max_retries: u32, + spent: u32, +} + +impl<'a> OllpRetryBudget<'a> { + /// A budget of `max_retries` retries on `predicate_class`. + pub fn new(orchestrator: &'a OllpOrchestrator, predicate_class: u64, max_retries: u32) -> Self { + Self { + orchestrator, + predicate_class, + max_retries, + spent: 0, + } + } + + /// A budget of the orchestrator's configured retries on + /// `predicate_class`. + pub fn configured(orchestrator: &'a OllpOrchestrator, predicate_class: u64) -> Self { + let max_retries = u32::from(orchestrator.ollp_max_retries()); + Self::new(orchestrator, predicate_class, max_retries) + } + + /// Whether every retry is spent. + pub fn exhausted(&self) -> bool { + self.spent >= self.max_retries + } + + /// The budget's size, as an exhaustion error reports it. + pub fn max_retries(&self) -> u8 { + u8::try_from(self.max_retries).unwrap_or(u8::MAX) + } + + /// Spend one retry: the orchestrator records it in the predicate class's + /// circuit window and backs off before the caller retries. + pub async fn spend(&mut self) { + self.orchestrator + .on_retry_required(self.predicate_class, self.spent) + .await; + self.spent += 1; + } +} diff --git a/nodedb/src/control/planner/calvin/retry_loop.rs b/nodedb/src/control/planner/calvin/retry_loop.rs index c11a52810..dab711606 100644 --- a/nodedb/src/control/planner/calvin/retry_loop.rs +++ b/nodedb/src/control/planner/calvin/retry_loop.rs @@ -14,6 +14,7 @@ use nodedb_cluster::calvin::{AbortReason, AttemptOutcome, CalvinCompletionRegist use crate::control::cluster::calvin::executor::ollp::error::OllpError; use crate::control::cluster::calvin::executor::ollp::orchestrator::OllpOrchestrator; use crate::control::planner::calvin::abort_error::calvin_abort_error; +use crate::control::planner::calvin::retry_budget::OllpRetryBudget; use crate::control::planner::calvin::submit::RoutedAssignment; use crate::control::planner::calvin::submit::budget::budget_ms_until; use crate::control::planner::calvin::submit::wait::await_completion; @@ -102,7 +103,7 @@ where mut rescan, } = args; let mut predicted = initial_predicted; - let mut retry: u32 = 0; + let mut budget = OllpRetryBudget::new(orchestrator, predicate_class_hash, ollp_max_retries); loop { budget_ms_until(deadline)?; let assignment = match submit(&predicted).await { @@ -125,16 +126,13 @@ where // PRE-ADMISSION failure (routed submit / circuit / budget). // Nothing executed, so there is no aborted attempt to re-scan // around — resubmit the SAME prediction after backoff. - if retry >= ollp_max_retries { + if budget.exhausted() { return Err(Error::OllpExhausted { - retries: ollp_max_retries.min(u8::MAX as u32) as u8, + retries: budget.max_retries(), cause: pre_admission_cause(ollp_err), }); } - orchestrator - .on_retry_required(predicate_class_hash, retry) - .await; - retry += 1; + budget.spend().await; continue; } }; @@ -179,16 +177,13 @@ where // participant released the attempt's locks before it acked, // so a FRESH reconnaissance is safe — and necessary, since the // stale prediction can never converge under drift. - if retry >= ollp_max_retries { + if budget.exhausted() { return Err(Error::OllpExhausted { - retries: ollp_max_retries.min(u8::MAX as u32) as u8, + retries: budget.max_retries(), cause: OllpExhaustedCause::PredicateDrift, }); } - orchestrator - .on_retry_required(predicate_class_hash, retry) - .await; - retry += 1; + budget.spend().await; predicted = rescan().await?; } } diff --git a/nodedb/src/control/planner/calvin/txn_predict.rs b/nodedb/src/control/planner/calvin/txn_predict.rs new file mode 100644 index 000000000..383a83d02 --- /dev/null +++ b/nodedb/src/control/planner/calvin/txn_predict.rs @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The prediction a predicate write inside a session transaction carries. +//! +//! A predicate `UPDATE` or `DELETE` in a transaction reads its rows at its +//! statement, as the transaction sees them. Its plan carries the matched +//! surrogates, and the implicit edges of the matched edge documents when the +//! collection is edge-bearing. The statement's stage refuses a prediction its +//! rows do not match. The buffered plan keeps the prediction, so a Calvin +//! COMMIT stages exactly those rows and checks them again at its turn. +//! +//! An update on an edge-bearing collection derives its implicit-edge +//! reconciliation from the same read, as the autocommit recon path does. + +use nodedb_physical::physical_plan::{DocumentOp, OllpPredictedEdge, PhysicalPlan, UpdateValue}; +use nodedb_physical::physical_task::PhysicalTask; + +use super::dependent_recon_plan::{inject_ollp_predicted_edges, inject_ollp_surrogates}; +use super::predicate::predicate_class_for_filters; +use super::preexec::{PreexecRequest, PreexecScan, RowIdentityRead, run_preexec_scan}; +use crate::control::planner::implicit_edges::{ + EdgeUpdateCtx, append_implicit_edge_update_tasks, parse_edge_field_overrides, +}; +use crate::control::state::SharedState; +use crate::types::{TraceId, TxnId}; + +/// The read a predicate write in a transaction took at its statement. +pub struct TxnPredictedRead { + /// The matched rows, as the transaction saw them. + pub scan: PreexecScan, + /// The implicit-edge tasks an update on an edge-bearing collection + /// derives from the read. Empty for a delete and for any other update. + pub edge_tasks: Vec, + /// The predicate class the write's retries are budgeted on. + pub predicate_class: u64, +} + +/// The predicate-write shape of a plan. +enum PredicateWrite<'a> { + Delete { + declared_primary_key: Option, + }, + Update { + updates: &'a [(String, UpdateValue)], + }, +} + +/// Read the rows the predicate write `task` matches in transaction `txn_id`, +/// and inject them into its plan. `None` for any other task. +/// +/// A delete on an edge-bearing collection also reads each row's identity, +/// which its node-delete edge tasks key on. +pub async fn predict_txn_bulk_write( + state: &SharedState, + task: &mut PhysicalTask, + txn_id: Option, +) -> crate::Result> { + let Some((collection, filters, shape)) = predicate_write(&task.plan) else { + return Ok(None); + }; + let (tenant_id, database_id) = (task.tenant_id, task.database_id); + let predicate_class = predicate_class_for_filters(&filters, &collection); + let edge_bearing = super::edge_truncate::collection_is_edge_bearing( + state, + tenant_id, + database_id, + &collection, + )?; + let identity = match &shape { + PredicateWrite::Delete { + declared_primary_key, + } if edge_bearing => RowIdentityRead::Read { + declared_primary_key: declared_primary_key.as_deref(), + }, + PredicateWrite::Delete { .. } | PredicateWrite::Update { .. } => RowIdentityRead::Skip, + }; + let scan = run_preexec_scan( + state, + tenant_id, + database_id, + PreexecRequest { + collection: &collection, + filters, + prefilter: None, + identity, + txn_id, + }, + ) + .await?; + + let mut edge_tasks = Vec::new(); + if edge_bearing && let PredicateWrite::Update { updates } = &shape { + let overrides = parse_edge_field_overrides(updates)?; + append_implicit_edge_update_tasks( + EdgeUpdateCtx { + state, + tenant_id, + database_id, + trace_id: TraceId::ZERO, + collection: &collection, + }, + &mut edge_tasks, + &scan.edges, + &scan.surrogates, + &overrides, + ) + .await?; + } + + inject_ollp_surrogates(&mut task.plan, scan.surrogates.clone()); + if edge_bearing { + let edges = scan.edges.iter().map(OllpPredictedEdge::from).collect(); + inject_ollp_predicted_edges(&mut task.plan, edges); + } + Ok(Some(TxnPredictedRead { + scan, + edge_tasks, + predicate_class, + })) +} + +/// The collection, serialized filters and shape of a predicate write. `None` +/// for any other plan. +fn predicate_write(plan: &PhysicalPlan) -> Option<(String, Vec, PredicateWrite<'_>)> { + let op = match plan { + PhysicalPlan::Document(op) => op, + PhysicalPlan::Vector(_) + | PhysicalPlan::Graph(_) + | PhysicalPlan::Kv(_) + | PhysicalPlan::Text(_) + | PhysicalPlan::Columnar(_) + | PhysicalPlan::Timeseries(_) + | PhysicalPlan::Spatial(_) + | PhysicalPlan::Crdt(_) + | PhysicalPlan::Query(_) + | PhysicalPlan::Meta(_) + | PhysicalPlan::Array(_) + | PhysicalPlan::ClusterArray(_) + | PhysicalPlan::ClusterEvent(_) => return None, + }; + match op { + DocumentOp::BulkDelete { + collection, + filters, + declared_primary_key, + .. + } => Some(( + collection.to_string(), + filters.clone(), + PredicateWrite::Delete { + declared_primary_key: declared_primary_key.clone(), + }, + )), + DocumentOp::BulkUpdate { + collection, + filters, + updates, + .. + } => Some(( + collection.to_string(), + filters.clone(), + PredicateWrite::Update { updates }, + )), + DocumentOp::PointGet { .. } + | DocumentOp::PointPut { .. } + | DocumentOp::PointInsert { .. } + | DocumentOp::PointDelete { .. } + | DocumentOp::PointUpdate { .. } + | DocumentOp::Upsert { .. } + | DocumentOp::BatchUpsert { .. } + | DocumentOp::BatchInsert { .. } + | DocumentOp::Scan { .. } + | DocumentOp::RangeScan { .. } + | DocumentOp::Register { .. } + | DocumentOp::IndexLookup { .. } + | DocumentOp::IndexedFetch { .. } + | DocumentOp::DropIndex { .. } + | DocumentOp::BackfillIndex { .. } + | DocumentOp::Truncate { .. } + | DocumentOp::EstimateCount { .. } + | DocumentOp::InsertSelect { .. } + | DocumentOp::UpdateFromJoin { .. } + | DocumentOp::Merge { .. } + | DocumentOp::ResolveWrite(_) + | DocumentOp::MaterializeScan { .. } + | DocumentOp::ResolvedWrite { .. } + | DocumentOp::ApplyBalanceDelta { .. } => None, + } +} diff --git a/nodedb/src/control/planner/rls_injection/document.rs b/nodedb/src/control/planner/rls_injection/document.rs index d4943d9ff..9e2d2e8c0 100644 --- a/nodedb/src/control/planner/rls_injection/document.rs +++ b/nodedb/src/control/planner/rls_injection/document.rs @@ -651,6 +651,7 @@ mod tests { projection: Vec::new(), limit: 0, offset: 0, + system_as_of_ms: None, }) } diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema/build/schema.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema/build/schema.rs index f2af6f037..215398149 100644 --- a/nodedb/src/control/planner/sql_plan_convert/output_schema/build/schema.rs +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema/build/schema.rs @@ -12,7 +12,6 @@ use std::collections::HashMap; use nodedb_sql::SqlError; use nodedb_sql::catalog::SqlCatalog; -use nodedb_sql::resolver::array_tvf::ARRAY_HORIZON_COLUMN; use nodedb_sql::types::query::Projection; use nodedb_sql::types::{ CtePlan, DocumentIndexLookupPlan, HybridSearchPlan, HybridSearchTriplePlan, InsertPlan, @@ -308,17 +307,9 @@ pub fn build_output_schema( // the row's `truncated_before_horizon` flag. SqlPlan::ArrayAgg(agg) => { let view = array_view(catalog, &agg.name)?; - let mut schema = array_columns_schema( - &nodedb_sql::resolver::array_tvf::array_agg_columns(&view, agg)?, - ); - if agg.temporal.is_temporal() { - schema.columns.push(OutputColumn { - display_name: ARRAY_HORIZON_COLUMN.to_string(), - lookup_key: ARRAY_HORIZON_COLUMN.to_string(), - ty: DdlColType::Bool, - }); - } - schema + array_columns_schema(&nodedb_sql::resolver::array_tvf::array_agg_row_columns( + &view, agg, + )?) } // `ArrayElementwise` compiles to `ArrayOp::Elementwise`, which // `describe_plan` classifies as `PlanKind::MultiRow`. `MultiRow` diff --git a/nodedb/src/control/planner/sql_plan_convert/scan/core.rs b/nodedb/src/control/planner/sql_plan_convert/scan/core.rs index cc7fbaebd..259345c01 100644 --- a/nodedb/src/control/planner/sql_plan_convert/scan/core.rs +++ b/nodedb/src/control/planner/sql_plan_convert/scan/core.rs @@ -229,6 +229,9 @@ pub(in crate::control::planner::sql_plan_convert) struct DocumentIndexLookupArgs pub offset: usize, pub tenant_id: TenantId, pub database_id: crate::types::DatabaseId, + /// The read's time scope. Only the current state and a system-time + /// cutoff plan an index lookup. + pub temporal: &'a nodedb_sql::TemporalScope, } pub(in crate::control::planner::sql_plan_convert) fn convert_document_index_lookup( @@ -244,7 +247,24 @@ pub(in crate::control::planner::sql_plan_convert) fn convert_document_index_look offset, tenant_id, database_id, + temporal, } = args; + // The versioned index answers at a system-time cutoff. It holds no valid + // time and lists no history, so the planner never routes those here. + if !matches!(temporal.valid_time, nodedb_sql::ValidTime::Any) { + return Err(crate::Error::PlanError { + detail: format!("index lookup on '{collection}' cannot answer a valid-time read"), + }); + } + let system_as_of_ms = match temporal.system_time { + SystemTimeScope::Current => None, + SystemTimeScope::AsOf(ms) => Some(ms), + SystemTimeScope::AllVersions => { + return Err(crate::Error::PlanError { + detail: format!("index lookup on '{collection}' cannot list every version"), + }); + } + }; let collection_key = nodedb_types::CollectionKey::from_bare(database_id, collection); let qualified_collection = nodedb_types::QualifiedCollection::new(database_id, collection); let filter_bytes = serialize_filters(filters)?; @@ -259,6 +279,7 @@ pub(in crate::control::planner::sql_plan_convert) fn convert_document_index_look // No LIMIT fetches every matching row under the byte budget. limit: limit.unwrap_or(usize::MAX), offset, + system_as_of_ms, }); Ok(vec![PhysicalTask { tenant_id, diff --git a/nodedb/src/control/planner/sql_plan_convert/scan/join.rs b/nodedb/src/control/planner/sql_plan_convert/scan/join.rs index 236f3ea77..e37fe71ad 100644 --- a/nodedb/src/control/planner/sql_plan_convert/scan/join.rs +++ b/nodedb/src/control/planner/sql_plan_convert/scan/join.rs @@ -102,6 +102,9 @@ fn bitmap_hint_to_plan(hint: &BitmapHint, database_id: DatabaseId) -> Option, + expected: usize, + appended: Mutex, + all_appended: Condvar, + } + + impl AuditWalSink for AppendGatedSink { + fn append_batch(&self, entries: &[AuditEntry]) -> crate::Result> { + let positions = AuditWalSink::append_batch(&*self.wal, entries)?; + let mut appended = self.appended.lock().unwrap_or_else(|p| p.into_inner()); + *appended += entries.len(); + self.all_appended.notify_all(); + Ok(positions) + } + + fn sync_through(&self, position: u64) -> crate::Result<()> { + let appended = self.appended.lock().unwrap_or_else(|p| p.into_inner()); + let (appended, _) = self + .all_appended + .wait_timeout_while(appended, Duration::from_secs(30), |appended| { + *appended < self.expected + }) + .unwrap_or_else(|p| p.into_inner()); + if *appended < self.expected { + return Err(crate::Error::Internal { + detail: format!( + "only {} of {} recorders appended within 30s; a sync wait holds the audit lock", + *appended, self.expected + ), + }); + } + drop(appended); + AuditWalSink::sync_through(&*self.wal, position) + } + + fn truncate_through(&self, position: u64) -> crate::Result<()> { + AuditWalSink::truncate_through(&*self.wal, position) + } + } + #[test] fn concurrent_recorders_share_fsyncs_and_keep_seq_order() { let dir = tempfile::tempdir().expect("tempdir"); let wal = Arc::new(WalManager::open_for_testing(&dir.path().join("group.wal")).expect("WAL")); + let sink = Arc::new(AppendGatedSink { + wal: Arc::clone(&wal), + expected: RECORDERS, + appended: Mutex::new(0), + all_appended: Condvar::new(), + }); let audit = Arc::new(Mutex::new(AuditLog::new(1_000))); - lock_audit_log(&audit).set_wal_sink(Arc::clone(&wal) as Arc); + lock_audit_log(&audit).set_wal_sink(sink as Arc); let syncs_before = wal.audit_sync_count(); - let start = Arc::new(Barrier::new(32)); + let start = Arc::new(Barrier::new(RECORDERS)); - let threads: Vec<_> = (0..32) + let threads: Vec<_> = (0..RECORDERS) .map(|i| { let (audit, start) = (Arc::clone(&audit), Arc::clone(&start)); std::thread::spawn(move || { @@ -248,16 +302,18 @@ mod tests { thread.join().expect("recorder thread").expect("durable"); } + // The first leader syncs every record appended so far, which is all of them. + // Every other recorder finds its LSN covered and issues no sync. let syncs = wal.audit_sync_count() - syncs_before; - assert!( - syncs < 32, - "32 single recorders must share fsyncs, saw {syncs}" + assert_eq!( + syncs, 1, + "{RECORDERS} recorders staged before any sync must share one fsync, saw {syncs}" ); let log = lock_audit_log(&audit); let published: Vec = log.all().iter().map(|e| e.seq).collect(); assert_eq!( published, - (1..=32).collect::>(), + (1..=RECORDERS as u64).collect::>(), "every entry is published once" ); drop(log); diff --git a/nodedb/src/control/security/catalog/surrogate_pk.rs b/nodedb/src/control/security/catalog/surrogate_pk.rs index c72f8a145..79cc404b0 100644 --- a/nodedb/src/control/security/catalog/surrogate_pk.rs +++ b/nodedb/src/control/security/catalog/surrogate_pk.rs @@ -79,6 +79,39 @@ impl SystemCatalog { Ok(found) } + /// The PK bound to each of `surrogates` in `key` under `tenant_id`, in + /// `surrogates` order, read in one read transaction. `None` for a + /// surrogate with no binding. + pub fn get_pks_for_surrogates( + &self, + key: CollectionKey<'_>, + tenant_id: TenantId, + surrogates: &[Surrogate], + ) -> crate::Result>>> { + if surrogates.is_empty() { + return Ok(Vec::new()); + } + let txn = self + .db + .begin_read() + .map_err(|e| catalog_err("surrogate_pk_rev batch read txn", e))?; + let Some(scope) = read_scope(&txn, key, tenant_id)? else { + return Ok(vec![None; surrogates.len()]); + }; + let table = txn + .open_table(SURROGATE_PK_REV) + .map_err(|e| catalog_err("open surrogate_pk_rev", e))?; + surrogates + .iter() + .map(|surrogate| { + Ok(table + .get(rev_key(scope, surrogate.as_u32())) + .map_err(|e| catalog_err("get surrogate_pk_rev", e))? + .map(|v| v.value().to_vec())) + }) + .collect() + } + /// Remove a surrogate ↔ PK binding atomically. Idempotent. pub fn delete_surrogate( &self, @@ -298,6 +331,33 @@ mod tests { ); } + /// One batch read answers every surrogate in order, `None` for an + /// unbound one, and an unknown collection answers `None` throughout. + #[test] + fn batch_reverse_read_answers_in_order() { + let (_dir, cat) = open_catalog(); + cat.put_surrogate(coll("users"), T0, b"alice", Surrogate::new(7)) + .unwrap(); + cat.put_surrogate(coll("users"), T0, b"bob", Surrogate::new(9)) + .unwrap(); + let wanted = [Surrogate::new(9), Surrogate::new(8), Surrogate::new(7)]; + assert_eq!( + cat.get_pks_for_surrogates(coll("users"), T0, &wanted) + .unwrap(), + vec![Some(b"bob".to_vec()), None, Some(b"alice".to_vec())] + ); + assert_eq!( + cat.get_pks_for_surrogates(coll("nobody"), T0, &wanted) + .unwrap(), + vec![None, None, None] + ); + assert!( + cat.get_pks_for_surrogates(coll("users"), T0, &[]) + .unwrap() + .is_empty() + ); + } + /// An overwrite to a new surrogate leaves no reverse row under the old /// one, so the two directions stay inverse. #[test] diff --git a/nodedb/src/control/server/exchange/resolve/exchange/post_process_arm.rs b/nodedb/src/control/server/exchange/resolve/exchange/post_process_arm.rs index 1de45ffee..7ab00b1a9 100644 --- a/nodedb/src/control/server/exchange/resolve/exchange/post_process_arm.rs +++ b/nodedb/src/control/server/exchange/resolve/exchange/post_process_arm.rs @@ -7,6 +7,7 @@ use nodedb_physical::physical_plan::{ ExchangeMode, ExchangeOp, GatheredSource, PhysicalPlan, QueryOp, SortKeySpec, TextOp, VectorOp, plan_contains_cluster_partitioned_leaf, }; +use nodedb_types::Surrogate; use super::super::array_read_rows::is_cluster_array_read; @@ -16,14 +17,13 @@ use crate::control::server::exchange::gather_outcome::GatherOutcome; use crate::control::server::exchange::owning_core::gather_single_owning_core; use crate::control::server::exchange::resolve::capture::DistributedReadCapture; use crate::control::server::response_translate::hit_key::parse_surrogate_hex; -use crate::control::server::response_translate::vector::{ - filter_vector_hits, resolve_surrogate_pk, -}; +use crate::control::server::response_translate::pk_resolve::resolve_surrogate_pks; +use crate::control::server::response_translate::vector::filter_vector_hits; use crate::control::server::shared::authorization::gathered_sources; use crate::control::state::SharedState; use crate::data::executor::response_codec::{ flatten_hybrid_hits_to_relational_rows, flatten_to_relational_rows, - flatten_vector_hits_to_relational_rows, + flatten_vector_hits_to_relational_rows, hybrid_hit_surrogates, vector_hit_surrogates, }; use crate::control::server::exchange::read_scope::ReadScope; @@ -290,21 +290,23 @@ pub(super) async fn materialize_child_rows( Some((rls_filters, top_k)) => filter_vector_hits(&merged, rls_filters, *top_k)?, None => merged, }; + // Every surrogate the flatten can ask for resolves in one batch. + let wanted = vector_hit_surrogates(&merged) + .into_iter() + .map(Surrogate::new); + let keys = resolve_surrogate_pks(state, database_id, tenant_id, &coll, wanted).await?; flatten_vector_hits_to_relational_rows(&merged, |surrogate| { - resolve_surrogate_pk( - state, - database_id, - tenant_id, - &coll, - nodedb_types::Surrogate::new(surrogate), - ) - }) + keys.get(Surrogate::new(surrogate)).map(str::to_owned) + })? } HitShape::Hybrid => { let merged = outcome.merged_rows(as_aggregate)?; + let wanted = hybrid_hit_surrogates(&merged) + .into_iter() + .map(Surrogate::new); + let keys = resolve_surrogate_pks(state, database_id, tenant_id, &coll, wanted).await?; flatten_hybrid_hits_to_relational_rows(&merged, |key| { - let surrogate = parse_surrogate_hex(key)?; - resolve_surrogate_pk(state, database_id, tenant_id, &coll, surrogate) + keys.get(parse_surrogate_hex(key)?).map(str::to_owned) }) } HitShape::ArrayRows => outcome.merged_array()?, diff --git a/nodedb/src/control/server/graph_dispatch/rag_fusion/coord.rs b/nodedb/src/control/server/graph_dispatch/rag_fusion/coord.rs index 4c0f32f54..bc1ba1349 100644 --- a/nodedb/src/control/server/graph_dispatch/rag_fusion/coord.rs +++ b/nodedb/src/control/server/graph_dispatch/rag_fusion/coord.rs @@ -28,7 +28,7 @@ use std::collections::{HashMap, HashSet}; use nodedb_physical::physical_plan::{GraphOp, RagStage}; -use nodedb_types::{RowIdentity, Surrogate}; +use nodedb_types::Surrogate; use crate::bridge::envelope::{Payload, PhysicalPlan, Response}; use crate::control::server::dispatch_utils::{not_found_response, ok_payload_response}; @@ -37,6 +37,7 @@ use crate::data::executor::handlers::graph_rag::{ RagResponseParams, graph_nodes_to_ranked_results, rag_response_body, vector_ranked_list, }; use crate::data::executor::handlers::graph_rag_triple::text_ranked_list; +use crate::data::executor::handlers::rag_key::RagKey; use crate::engine::graph::edge_store::Direction; use crate::engine::graph::traversal_options::GraphTraversalOptions; use crate::query::fusion::reciprocal_rank_fusion_weighted; @@ -134,12 +135,18 @@ async fn run(state: &SharedState, scope: RagScope, plan: &PhysicalPlan) -> crate }; let legs = exported.legs; - // 2. Name the hits' graph nodes. + // 2. Name the hits' graph nodes. Text hits key by their graph node too, + // as on one core, but only vector hits seed the walk. let mut hit_surrogates: Vec = Vec::new(); - for hit in &legs.vector { - if let Some(raw) = hit.surrogate - && !hit_surrogates.contains(&raw) - { + let mut requested: HashSet = HashSet::new(); + let text_surrogates = legs.text.iter().map(|hit| hit.surrogate); + for raw in legs + .vector + .iter() + .filter_map(|hit| hit.surrogate) + .chain(text_surrogates) + { + if requested.insert(raw) { hit_surrogates.push(raw); } } @@ -160,18 +167,18 @@ async fn run(state: &SharedState, scope: RagScope, plan: &PhysicalPlan) -> crate ) .await? }; - let mut vector_scores: HashMap = HashMap::new(); + let mut vector_scores: HashMap = HashMap::new(); let mut seeds: Vec = Vec::new(); for (rank, hit) in legs.vector.iter().enumerate() { let key = match hit.surrogate { - Some(raw) => match seed_bindings.by_surrogate.get(&raw) { - Some(name) => { - seeds.push(name.clone()); - RowIdentity::from_user_key(name.clone()) + Some(raw) => { + let name = seed_bindings.by_surrogate.get(&raw).map(String::as_str); + if let Some(name) = name { + seeds.push(name.to_owned()); } - None => RowIdentity::for_surrogate(Surrogate::new(raw)), - }, - None => RowIdentity::from_user_key(format!("__unbound_{}", hit.entry_id)), + RagKey::of_hit(name, Surrogate::new(raw)) + } + None => RagKey::Unbound(hit.entry_id), }; vector_scores.insert(key, (rank, hit.distance)); } @@ -231,7 +238,12 @@ async fn run(state: &SharedState, scope: RagScope, plan: &PhysicalPlan) -> crate .iter() .map(|hit| (Surrogate::new(hit.surrogate), hit.score)) .collect(); - let text_list = text_ranked_list(&text_hits); + let text_list = text_ranked_list(&text_hits, |s| { + seed_bindings + .by_surrogate + .get(&s.as_u32()) + .map(String::as_str) + }); ( reciprocal_rank_fusion_weighted( &[vector_list, text_list, graph_list], diff --git a/nodedb/src/control/server/http/routes/query/materialized/append.rs b/nodedb/src/control/server/http/routes/query/materialized/append.rs index 71e2b86cd..124057225 100644 --- a/nodedb/src/control/server/http/routes/query/materialized/append.rs +++ b/nodedb/src/control/server/http/routes/query/materialized/append.rs @@ -46,7 +46,7 @@ pub(super) struct ShapedAppend<'a> { } /// Shape a response. A refusal becomes the HTTP error its code maps to. -pub(super) fn append_response( +pub(super) async fn append_response( result_rows: &mut Vec, response: crate::bridge::envelope::Response, append: &ShapedAppend<'_>, @@ -54,12 +54,12 @@ pub(super) fn append_response( if response.status != Status::Ok { return Err(response_error(&response)); } - append_payload(result_rows, &response.payload.to_vec(), append) + append_payload(result_rows, &response.payload.to_vec(), append).await } /// Shape one payload into rows, or pass a write's or tag's payload through /// as one JSON row. An empty payload adds nothing. -pub(super) fn append_payload( +pub(super) async fn append_payload( result_rows: &mut Vec, payload: &[u8], append: &ShapedAppend<'_>, @@ -85,7 +85,9 @@ pub(super) fn append_payload( tenant_id: append.tenant_id, redaction: Some(append.redaction.ctx(&append.state.shared.redaction)), sequences: Some(&sequences), - }) { + }) + .await + { Ok(HttpShaped::Rows(rows)) => result_rows.extend(rows), Ok(HttpShaped::Passthrough) => result_rows.push(passthrough_json_row(payload)), Err(e) => return Err(shape_error_to_api(e)), diff --git a/nodedb/src/control/server/http/routes/query/materialized/shape.rs b/nodedb/src/control/server/http/routes/query/materialized/shape.rs index 98d84299e..b0cea73ae 100644 --- a/nodedb/src/control/server/http/routes/query/materialized/shape.rs +++ b/nodedb/src/control/server/http/routes/query/materialized/shape.rs @@ -83,7 +83,7 @@ pub(super) async fn run_task_loop( }; if let Some(orchestrated) = run_orchestrated(&state.shared, identity, &task).await? { - append_payload(&mut result_rows, &orchestrated.payload, &append)?; + append_payload(&mut result_rows, &orchestrated.payload, &append).await?; if orchestrated.metered { meter_task_dispatch( &state.shared, @@ -115,7 +115,7 @@ pub(super) async fn run_task_loop( .map_err(gateway_error)? { crate::control::server::shared::clone_write::CloneCheckedOutcome::Handled(resp) => { - append_response(&mut result_rows, resp, &append)?; + append_response(&mut result_rows, resp, &append).await?; meter_task_dispatch( &state.shared, &scope, @@ -144,7 +144,7 @@ pub(super) async fn run_task_loop( .await .map_err(gateway_error)?; for payload in &payloads { - append_payload(&mut result_rows, payload, &append)?; + append_payload(&mut result_rows, payload, &append).await?; } meter_task_dispatch( &state.shared, diff --git a/nodedb/src/control/server/http/routes/query/ndjson.rs b/nodedb/src/control/server/http/routes/query/ndjson.rs index e93fa3768..f9be90789 100644 --- a/nodedb/src/control/server/http/routes/query/ndjson.rs +++ b/nodedb/src/control/server/http/routes/query/ndjson.rs @@ -32,7 +32,7 @@ use super::super::super::peer::PeerAddr; use super::super::super::transport::ClientTransport; use super::super::atomic_statement::{RoutedStatement, route_http_statement}; use super::super::query_stream::{NdjsonBody, ndjson_body_stream, try_open_stream}; -use super::super::result_shape::{HttpShaped, passthrough_to_ndjson, shape_http_payload}; +use super::super::result_shape::append_ndjson_payload; use super::{DatabaseQueryParam, resolve_database_id}; /// POST /v1/query/stream — execute SQL, return results as NDJSON. @@ -444,33 +444,21 @@ pub async fn query_ndjson( if payload.is_empty() { continue; } - match shape_http_payload(MaterializedShapeRequest { - payload, - plan: &plan_for_shape, - plan_kind, - projection: Some(&output_schema), - state: &state.shared, - database_id, - tenant_id, - redaction: Some(redaction.ctx(&state.shared.redaction)), - sequences: Some(&sequences), - }) { - Ok(HttpShaped::Rows(rows)) => { - task_rows += rows.len() as u64; - for row in rows { - ndjson.push_str(&row.to_string()); - ndjson.push('\n'); - } - } - Ok(HttpShaped::Passthrough) => { - task_rows += 1; - passthrough_to_ndjson(payload, &mut ndjson); - } - Err(e) => { - ndjson.push_str(&serde_json::json!({"error": e.message()}).to_string()); - ndjson.push('\n'); - } - } + task_rows += append_ndjson_payload( + &mut ndjson, + MaterializedShapeRequest { + payload, + plan: &plan_for_shape, + plan_kind, + projection: Some(&output_schema), + state: &state.shared, + database_id, + tenant_id, + redaction: Some(redaction.ctx(&state.shared.redaction)), + sequences: Some(&sequences), + }, + ) + .await; } if let Some(info) = &plan_metering_info { meter_dispatch(&state.shared, &scope, info, Some(task_rows)); diff --git a/nodedb/src/control/server/http/routes/result_shape.rs b/nodedb/src/control/server/http/routes/result_shape.rs index e217ff02d..bda23af70 100644 --- a/nodedb/src/control/server/http/routes/result_shape.rs +++ b/nodedb/src/control/server/http/routes/result_shape.rs @@ -51,10 +51,10 @@ pub(super) enum HttpShaped { /// Shape one Data-Plane payload for HTTP, mirroring the pgwire/native /// materialized-shaping call site. -pub(super) fn shape_http_payload( +pub(super) async fn shape_http_payload( request: MaterializedShapeRequest<'_>, ) -> Result { - match shape_response_materialized(request)? { + match shape_response_materialized(request).await? { // Each row map is already keyed by `ShapedRows::cell_keys`, so it // serializes to JSON key for key, each typed cell converted at this // edge. When two output columns share a name the later one carries a @@ -105,6 +105,35 @@ pub(super) fn passthrough_to_ndjson(payload: &[u8], ndjson: &mut String) { } } +/// Shape one payload and append it to an NDJSON buffer: one line per row, a +/// passthrough payload as [`passthrough_to_ndjson`] writes it, or one error +/// line when shaping fails. Returns the rows it counts for metering. +pub(super) async fn append_ndjson_payload( + ndjson: &mut String, + request: MaterializedShapeRequest<'_>, +) -> u64 { + let payload = request.payload; + match shape_http_payload(request).await { + Ok(HttpShaped::Rows(rows)) => { + let count = rows.len() as u64; + for row in rows { + ndjson.push_str(&row.to_string()); + ndjson.push('\n'); + } + count + } + Ok(HttpShaped::Passthrough) => { + passthrough_to_ndjson(payload, ndjson); + 1 + } + Err(e) => { + ndjson.push_str(&serde_json::json!({"error": e.message()}).to_string()); + ndjson.push('\n'); + 0 + } + } +} + /// Render protocol-neutral DDL results to JSON rows. /// /// Status and empty results keep their prior JSON shapes (`{type, tag}` / diff --git a/nodedb/src/control/server/native/dispatch/direct_orchestrated.rs b/nodedb/src/control/server/native/dispatch/direct_orchestrated.rs index ef055c4c9..e1f1133a5 100644 --- a/nodedb/src/control/server/native/dispatch/direct_orchestrated.rs +++ b/nodedb/src/control/server/native/dispatch/direct_orchestrated.rs @@ -56,7 +56,7 @@ pub(super) async fn dispatch_orchestrated( crate::control::insert_select::run_authorized_insert_select(ctx.state, authorized) .await; return Some(match result { - Ok(resp) => data_plane_response_to_native(ctx, seq, plan, &resp), + Ok(resp) => data_plane_response_to_native(ctx, seq, plan, &resp).await, Err(e) => error_to_native(seq, &e), }); } @@ -86,7 +86,7 @@ pub(super) async fn dispatch_orchestrated( crate::control::merge_orchestrator::run_authorized_merge(ctx.state, authorized) .await; return Some(match result { - Ok(resp) => data_plane_response_to_native(ctx, seq, plan, &resp), + Ok(resp) => data_plane_response_to_native(ctx, seq, plan, &resp).await, Err(e) => error_to_native(seq, &e), }); } @@ -116,7 +116,7 @@ pub(super) async fn dispatch_orchestrated( ) .await; return Some(match result { - Ok(resp) => data_plane_response_to_native(ctx, seq, plan, &resp), + Ok(resp) => data_plane_response_to_native(ctx, seq, plan, &resp).await, Err(e) => error_to_native(seq, &e), }); } @@ -136,7 +136,7 @@ pub(super) async fn dispatch_orchestrated( ) .await; return Some(match result { - Ok(resp) => data_plane_response_to_native(ctx, seq, plan, &resp), + Ok(resp) => data_plane_response_to_native(ctx, seq, plan, &resp).await, Err(e) => error_to_native(seq, &e), }); } diff --git a/nodedb/src/control/server/native/dispatch/graph_match.rs b/nodedb/src/control/server/native/dispatch/graph_match.rs index 94d404ab7..f7dda5697 100644 --- a/nodedb/src/control/server/native/dispatch/graph_match.rs +++ b/nodedb/src/control/server/native/dispatch/graph_match.rs @@ -133,7 +133,7 @@ pub(crate) async fn handle_graph_match( } if response.status == Status::Error { - return data_plane_response_to_native(ctx, seq, &plan_for_response, &response); + return data_plane_response_to_native(ctx, seq, &plan_for_response, &response).await; } // Unwrap the `{rows, frontier, resume}` envelope into a bare rows array. @@ -146,7 +146,7 @@ pub(crate) async fn handle_graph_match( }, Err(error) => return error_to_native(seq, &error), }; - let native = data_plane_response_to_native(ctx, seq, &plan_for_response, &unwrapped); + let native = data_plane_response_to_native(ctx, seq, &plan_for_response, &unwrapped).await; // Metered only on the success path — `data_plane_response_to_native` // above already decoded the row envelope to shape the response, so // `native.rows` gives the real row count for free (no extra decode). @@ -222,7 +222,8 @@ async fn match_across_shards( seq, plan, &crate::control::server::dispatch_utils::ok_payload_response(outcome.rows_payload), - ), + ) + .await, Err(error) => error_to_native(seq, &error), } } diff --git a/nodedb/src/control/server/native/dispatch/graph_owner.rs b/nodedb/src/control/server/native/dispatch/graph_owner.rs index 792cf55d0..e20509a74 100644 --- a/nodedb/src/control/server/native/dispatch/graph_owner.rs +++ b/nodedb/src/control/server/native/dispatch/graph_owner.rs @@ -150,7 +150,7 @@ pub(super) async fn dispatch_graph_owner_read( } }; Some(match served { - Ok(response) => data_plane_response_to_native(ctx, seq, plan, &response), + Ok(response) => data_plane_response_to_native(ctx, seq, plan, &response).await, Err(error) => error_to_native(seq, &error), }) } diff --git a/nodedb/src/control/server/native/dispatch/response.rs b/nodedb/src/control/server/native/dispatch/response.rs index c69f03090..944477fb3 100644 --- a/nodedb/src/control/server/native/dispatch/response.rs +++ b/nodedb/src/control/server/native/dispatch/response.rs @@ -17,7 +17,7 @@ use super::{ shape_error_to_native, to_native_columns_rows, }; -pub(crate) fn data_plane_response_to_native( +pub(crate) async fn data_plane_response_to_native( ctx: &DispatchCtx<'_>, seq: u64, plan: &PhysicalPlan, @@ -57,7 +57,9 @@ pub(crate) fn data_plane_response_to_native( redaction: Some(redaction.ctx(&ctx.state.redaction)), // No projection, so no Control-Plane computed column to resolve. sequences: None, - }) { + }) + .await + { Ok(ShapeOutcome::Rows(shaped)) => { let (columns, rows) = to_native_columns_rows(&shaped); NativeResponse { diff --git a/nodedb/src/control/server/native/dispatch/single_task.rs b/nodedb/src/control/server/native/dispatch/single_task.rs index ba28d0ed5..aca5d6763 100644 --- a/nodedb/src/control/server/native/dispatch/single_task.rs +++ b/nodedb/src/control/server/native/dispatch/single_task.rs @@ -126,7 +126,8 @@ pub(super) async fn dispatch_single_task( read_versions: crate::types::ReadVersions::new(), write_set: Vec::new(), }; - return data_plane_response_to_native(ctx, seq, &plan_for_staged_response, &synthetic); + return data_plane_response_to_native(ctx, seq, &plan_for_staged_response, &synthetic) + .await; } Err(StagingGateError::Dispatch(e)) => return error_to_native(seq, &e), Err(StagingGateError::Rejected { code }) => { @@ -221,7 +222,7 @@ pub(super) async fn dispatch_single_task( ) .await; } - let native = data_plane_response_to_native(ctx, seq, &plan_for_response, &resp); + let native = data_plane_response_to_native(ctx, seq, &plan_for_response, &resp).await; // Metered here, once this task's real dispatch has already // succeeded — covers the `Read` route (autocommit writes/reads, // in-transaction reads); see `plan_metering_info`'s doc comment diff --git a/nodedb/src/control/server/native/dispatch/sorted_read_op.rs b/nodedb/src/control/server/native/dispatch/sorted_read_op.rs index 29b9859f2..fce611375 100644 --- a/nodedb/src/control/server/native/dispatch/sorted_read_op.rs +++ b/nodedb/src/control/server/native/dispatch/sorted_read_op.rs @@ -56,7 +56,7 @@ pub(crate) async fn handle_sorted_read_op( ) .await { - Ok((plan, response)) => data_plane_response_to_native(ctx, seq, &plan, &response), + Ok((plan, response)) => data_plane_response_to_native(ctx, seq, &plan, &response).await, Err(error) => ddl_result_to_native(seq, Err(error)), } } diff --git a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs index 830703967..03364aed9 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/run.rs @@ -38,7 +38,7 @@ use super::super::execute_dml_hooks::{self, PreDispatchHandled}; use super::super::result_shaping::ResultShaping; use super::super::streaming::StreamSelectContext; use super::finish::StatementTail; -use super::task::ShapeTaskParams; +use super::task::{ShapeTaskParams, staged_rows_response}; use super::tracking::DispatchedTask; use super::txn_task::StatementTxnOutcome; use crate::control::server::shared::txn_route::{StatementEvents, TxnTaskContext}; @@ -198,7 +198,8 @@ impl NodeDbPgHandler { &mut responses, &mut returning_rows, &mut statement_tag, - )?; + ) + .await?; } continue; } @@ -446,7 +447,8 @@ impl NodeDbPgHandler { &mut responses, &mut returning_rows, &mut statement_tag, - )? + ) + .await? }; // Metered here, once per successfully dispatched task — every @@ -484,19 +486,3 @@ impl NodeDbPgHandler { Ok(responses) } } - -/// A staged write's `RETURNING` rows as the response the shaper reads. -fn staged_rows_response(rows: Vec) -> crate::bridge::envelope::Response { - crate::bridge::envelope::Response { - request_id: crate::types::RequestId::new(0), - status: crate::bridge::envelope::Status::Ok, - attempt: 0, - partial: false, - payload: crate::bridge::envelope::Payload::from_vec(rows), - watermark_lsn: crate::types::Lsn::ZERO, - error_code: None, - stage_vote: None, - read_versions: crate::types::ReadVersions::new(), - write_set: Vec::new(), - } -} diff --git a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/task.rs b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/task.rs index 7a9eed07f..4cd290b7d 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/task.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/dispatch_loop/task.rs @@ -53,7 +53,7 @@ impl NodeDbPgHandler { /// /// Returns the task's own row count for metering, `None` for a /// passthrough response with no row payload to count. - pub(super) fn shape_task_response( + pub(super) async fn shape_task_response( &self, params: ShapeTaskParams<'_>, responses: &mut Vec, @@ -91,6 +91,7 @@ impl NodeDbPgHandler { redaction: Some(redaction.ctx(&self.state.redaction)), sequences: Some(&sequences), }) + .await .map_err(|e| shape_error_to_pg(&e))? { ShapeOutcome::Rows(shaped) => { @@ -130,3 +131,19 @@ impl NodeDbPgHandler { } } } + +/// A staged write's `RETURNING` rows as the response the shaper reads. +pub(super) fn staged_rows_response(rows: Vec) -> crate::bridge::envelope::Response { + crate::bridge::envelope::Response { + request_id: crate::types::RequestId::new(0), + status: crate::bridge::envelope::Status::Ok, + attempt: 0, + partial: false, + payload: crate::bridge::envelope::Payload::from_vec(rows), + watermark_lsn: crate::types::Lsn::ZERO, + error_code: None, + stage_vote: None, + read_versions: crate::types::ReadVersions::new(), + write_set: Vec::new(), + } +} diff --git a/nodedb/src/control/server/pgwire/handler/routing/gateway_dispatch.rs b/nodedb/src/control/server/pgwire/handler/routing/gateway_dispatch.rs index 191bfacf4..b070902e6 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/gateway_dispatch.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/gateway_dispatch.rs @@ -209,14 +209,16 @@ impl NodeDbPgHandler { // The clone-write hook fully handled this task locally — // never forwarded — so its one payload folds exactly // like a forwarded one. - let rows = self.fold_gateway_payload( - &mut fold, - resp.payload.as_ref(), - plan_kind, - shape_plan.as_ref(), - tag_role, - &shaping, - )?; + let rows = self + .fold_gateway_payload( + &mut fold, + resp.payload.as_ref(), + plan_kind, + shape_plan.as_ref(), + tag_role, + &shaping, + ) + .await?; meter_gateway_task( &self.state, metering_scope.as_ref(), @@ -246,24 +248,29 @@ impl NodeDbPgHandler { // write is refused by the count reader. let mut task_rows: Option = None; if payloads.is_empty() { - task_rows = self.fold_gateway_payload( - &mut fold, - &[], - plan_kind, - shape_plan.as_ref(), - tag_role, - &shaping, - )?; + task_rows = self + .fold_gateway_payload( + &mut fold, + &[], + plan_kind, + shape_plan.as_ref(), + tag_role, + &shaping, + ) + .await?; } for payload in &payloads { - if let Some(rows) = self.fold_gateway_payload( - &mut fold, - payload, - plan_kind, - shape_plan.as_ref(), - tag_role, - &shaping, - )? { + if let Some(rows) = self + .fold_gateway_payload( + &mut fold, + payload, + plan_kind, + shape_plan.as_ref(), + tag_role, + &shaping, + ) + .await? + { task_rows = Some(task_rows.unwrap_or(0) + rows); } } diff --git a/nodedb/src/control/server/pgwire/handler/routing/gateway_fold.rs b/nodedb/src/control/server/pgwire/handler/routing/gateway_fold.rs index 94edbe118..11f016bb6 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/gateway_fold.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/gateway_fold.rs @@ -101,14 +101,14 @@ impl NodeDbPgHandler { /// does, through `shape_response_materialized` with the task's plan. A /// forwarded Data-Plane payload has the shape a local core produces, and /// the plan-dependent steps (the KV point-get `{key, value}` row wrap, the - /// vector surrogate-to-PK translation) must run on it too: shaped without + /// search surrogate-to-PK naming) must run on it too: shaped without /// them, a KV point read's stored value decodes as a scalar and yields no /// row. `shape_plan` is the task's plan, which a row-producing kind always /// carries (see [`plan_produces_rows`]). /// /// Returns the rows the payload decoded to, `None` for a passthrough /// payload with no row count. - pub(super) fn fold_gateway_payload( + pub(super) async fn fold_gateway_payload( &self, fold: &mut GatewayFold, payload: &[u8], @@ -133,6 +133,7 @@ impl NodeDbPgHandler { redaction: Some(shaping.redaction.ctx(&self.state.redaction)), sequences: None, }) + .await .map_err(|e| shape_error_to_pg(&e))?, (true, None) => { return Err(error_to_pg(&crate::Error::Internal { diff --git a/nodedb/src/control/server/response_shape/calvin_fold.rs b/nodedb/src/control/server/response_shape/calvin_fold.rs index 9ff74fdec..8897c4d72 100644 --- a/nodedb/src/control/server/response_shape/calvin_fold.rs +++ b/nodedb/src/control/server/response_shape/calvin_fold.rs @@ -10,7 +10,7 @@ use nodedb_physical::physical_plan::{DocumentOp, KvOp, PhysicalPlan}; -use super::compose::{ShapeOutcome, shape_response_materialized}; +use super::compose::{ShapeOutcome, shape_returning_materialized}; use super::redaction::QueryRedaction; use super::request::MaterializedShapeRequest; use super::schema::OutputSchema; @@ -115,7 +115,7 @@ pub fn calvin_task_outcome( let redaction = QueryRedaction::for_plan(ctx.tenant_id, ctx.database_id, ctx.auth, plan); if let (PlanKind::ReturningRows, Some(resp)) = (plan_kind, apply_resp) && let Ok(ShapeOutcome::Rows(shaped)) = - shape_response_materialized(MaterializedShapeRequest { + shape_returning_materialized(MaterializedShapeRequest { payload: resp.payload.as_bytes(), plan, plan_kind: PlanKind::ReturningRows, diff --git a/nodedb/src/control/server/response_shape/compose/materialized.rs b/nodedb/src/control/server/response_shape/compose/materialized.rs index 5ed6840f6..9bf118d15 100644 --- a/nodedb/src/control/server/response_shape/compose/materialized.rs +++ b/nodedb/src/control/server/response_shape/compose/materialized.rs @@ -50,10 +50,13 @@ pub enum ShapeOutcome { } /// Shape a single Data-Plane payload into protocol-neutral rows, applying -/// the canonical shaping order: KV point-get wrap, vector surrogate->PK -/// translation, payload decode, scan-envelope unwrap, and (when +/// the canonical shaping order: KV point-get wrap, walk wrap, search +/// surrogate->PK naming, payload decode, scan-envelope unwrap, and (when /// `projection` names columns) SELECT-list column selection. -pub fn shape_response_materialized( +/// +/// Async: the search naming step can ask a collection's home leader for +/// keys this node's catalog lacks. +pub async fn shape_response_materialized( request: MaterializedShapeRequest<'_>, ) -> Result { let MaterializedShapeRequest { @@ -79,21 +82,71 @@ pub fn shape_response_materialized( } // The plan-dependent transforms run before any decode or shape step: - // KV wrap, then walk wrap, then search translation. + // KV wrap, then walk wrap, then search naming. + let wrapped = apply_walk_wrap(plan, &apply_kv_wrap(plan, payload))?; + let translated = + translate_search_response(&wrapped, plan, state, database_id, tenant_id).await?; + shape_translated( + &translated, + plan_kind, + projection, + redaction, + sequences.map(|s| s as &dyn SequenceAccess), + ) +} + +/// Shape a write's `RETURNING` payload. Synchronous: a `RETURNING` payload +/// holds stored rows, never search hits, so it has no surrogate to name. +/// Any other plan kind is refused, since it can carry search hits. +pub fn shape_returning_materialized( + request: MaterializedShapeRequest<'_>, +) -> Result { + let MaterializedShapeRequest { + payload, + plan, + plan_kind, + projection, + redaction, + sequences, + .. + } = request; + if !matches!(plan_kind, PlanKind::ReturningRows) { + return Err(crate::Error::Internal { + detail: format!( + "returning shaping received a {plan_kind:?} payload, which needs search naming" + ), + } + .into()); + } let wrapped = apply_walk_wrap(plan, &apply_kv_wrap(plan, payload))?; - let translated = translate_search_response(&wrapped, plan, state, database_id, tenant_id); + shape_translated( + &wrapped, + plan_kind, + projection, + redaction, + sequences.map(|s| s as &dyn SequenceAccess), + ) +} +/// Shape a payload whose plan-dependent transforms already ran. +fn shape_translated( + translated: &[u8], + plan_kind: PlanKind, + projection: Option<&OutputSchema>, + redaction: Option>, + sequences: Option<&dyn SequenceAccess>, +) -> Result { let shaped = match plan_kind { - PlanKind::ArraySlice => shape_array_slice(&translated, projection, redaction, sequences)?, + PlanKind::ArraySlice => shape_array_slice(translated, projection, redaction, sequences)?, // `RETURNING` rows are held to the columns already announced to the // client, when any were — see `super::returning`. PlanKind::ReturningRows => { - shape_returning_rows(&translated, projection, redaction, sequences)? + shape_returning_rows(translated, projection, redaction, sequences)? } PlanKind::SingleDocument | PlanKind::MultiRow => { - shape_generic_rows(&translated, projection, redaction, sequences)? + shape_generic_rows(translated, projection, redaction, sequences)? } - // Handled by the early return above; kept exhaustive (no catch-all, + // The callers return early for these; kept exhaustive (no catch-all, // no panic) so a future PlanKind desync degrades to passthrough // rather than crashing the connection. PlanKind::Execution | PlanKind::DmlResult(_) | PlanKind::DmlResultByOp => { @@ -229,6 +282,7 @@ mod tests { redaction: None, sequences: None, }) + .await .expect("execution plan passthrough"); assert!(matches!(materialized, ShapeOutcome::Passthrough)); assert_eq!( diff --git a/nodedb/src/control/server/response_shape/compose/mod.rs b/nodedb/src/control/server/response_shape/compose/mod.rs index bf76a3773..ae00ff17a 100644 --- a/nodedb/src/control/server/response_shape/compose/mod.rs +++ b/nodedb/src/control/server/response_shape/compose/mod.rs @@ -8,4 +8,6 @@ pub mod kernel; pub mod materialized; pub use kernel::shape_decoded_rows; -pub use materialized::{ShapeOutcome, shape_payload_no_plan, shape_response_materialized}; +pub use materialized::{ + ShapeOutcome, shape_payload_no_plan, shape_response_materialized, shape_returning_materialized, +}; diff --git a/nodedb/src/control/server/response_shape/request.rs b/nodedb/src/control/server/response_shape/request.rs index 9e66c00c0..9c0e0ac32 100644 --- a/nodedb/src/control/server/response_shape/request.rs +++ b/nodedb/src/control/server/response_shape/request.rs @@ -35,6 +35,7 @@ pub struct MaterializedShapeRequest<'a> { /// Session-scoped sequence access for the projection's Control-Plane /// computed columns. `None` only where the producer has no session in /// scope; a projection that carries computed columns then fails rather - /// than shipping NULL under the alias. - pub sequences: Option<&'a dyn SequenceAccess>, + /// than shipping NULL under the alias. `Sync`, so an awaiting shaper + /// stays `Send` across the search translation. + pub sequences: Option<&'a (dyn SequenceAccess + Sync)>, } diff --git a/nodedb/src/control/server/response_translate/dispatch.rs b/nodedb/src/control/server/response_translate/dispatch.rs index d4e666e05..51e2c341e 100644 --- a/nodedb/src/control/server/response_translate/dispatch.rs +++ b/nodedb/src/control/server/response_translate/dispatch.rs @@ -8,61 +8,91 @@ use nodedb_types::{DatabaseId, TenantId}; use crate::bridge::envelope::PhysicalPlan; use crate::control::state::SharedState; -use nodedb_physical::physical_plan::{TextOp, VectorOp}; +use nodedb_physical::physical_plan::{GraphOp, QueryOp, RagStage, TextOp, VectorOp}; +use super::rag_fusion::translate_rag_fusion_payload; use super::text_hybrid::{translate_hybrid_search_payload, translate_text_search_payload}; use super::vector::{VectorHitCut, translate_vector_search_payload}; +/// Where a translator runs: the node's state and the request's scope. +#[derive(Clone, Copy)] +struct TranslateScope<'a> { + state: &'a SharedState, + database_id: DatabaseId, + tenant_id: TenantId, +} + /// Inspect the executed plan and apply the surrogate→PK translation that -/// matches its shape: vector search hits, full-text search hits, or hybrid -/// (RRF) fusion hits. Every other plan kind returns the payload untouched. -pub fn translate_search_response( +/// matches its shape: vector search hits, full-text search hits, hybrid +/// (RRF) fusion hits, or GraphRAG fusion rows. Every other plan kind returns +/// the payload untouched. +pub async fn translate_search_response( payload: &[u8], plan: &PhysicalPlan, state: &SharedState, database_id: DatabaseId, tenant_id: TenantId, -) -> Vec { - // The coordinator wraps sharded reads (including search) in an - // `Exchange{Gather}` node; unwrap it so the underlying op is visible and - // translation still runs on the gathered payload. - if let PhysicalPlan::Query(nodedb_physical::physical_plan::QueryOp::Exchange(op)) = plan { - return translate_search_response(payload, &op.child, state, database_id, tenant_id); +) -> crate::Result> { + // The coordinator wraps sharded reads (search included) in an + // `Exchange{Gather}` node. The underlying op decides the translation. + let mut plan = plan; + while let PhysicalPlan::Query(QueryOp::Exchange(op)) = plan { + plan = &*op.child; } - + let scope = TranslateScope { + state, + database_id, + tenant_id, + }; match plan { + PhysicalPlan::Vector(op) => translate_vector_op(payload, op, scope).await, + PhysicalPlan::Text(op) => translate_text_op(payload, op, scope).await, + PhysicalPlan::Graph(op) => translate_graph_op(payload, op, scope).await, + PhysicalPlan::Document(_) + | PhysicalPlan::Kv(_) + | PhysicalPlan::Columnar(_) + | PhysicalPlan::Timeseries(_) + | PhysicalPlan::Spatial(_) + | PhysicalPlan::Crdt(_) + | PhysicalPlan::Query(_) + | PhysicalPlan::Meta(_) + | PhysicalPlan::Array(_) + | PhysicalPlan::ClusterArray(_) + | PhysicalPlan::ClusterEvent(_) => Ok(payload.to_vec()), + } +} + +/// The vector search ops' translation. Every other vector op carries no hits. +async fn translate_vector_op( + payload: &[u8], + op: &VectorOp, + scope: TranslateScope<'_>, +) -> crate::Result> { + let (collection, cut) = match op { // A slow-path search attached each body for its column projection, // so the body stays after the filter. The fast path attached it only // for the predicate. - PhysicalPlan::Vector(VectorOp::Search { + VectorOp::Search { collection, rls_filters, top_k, skip_payload_fetch, .. - }) => translate_vector_search_payload( - payload, - state, - database_id, - tenant_id, - collection.as_str(), + } => ( + collection, VectorHitCut { rls_filters: rls_filters.as_slice(), top_k: *top_k, strip_bodies: *skip_payload_fetch, }, ), - PhysicalPlan::Vector(VectorOp::MultiSearch { + VectorOp::MultiSearch { collection, rls_filters, top_k, .. - }) => translate_vector_search_payload( - payload, - state, - database_id, - tenant_id, - collection.as_str(), + } => ( + collection, VectorHitCut { rls_filters: rls_filters.as_slice(), top_k: *top_k, @@ -71,58 +101,150 @@ pub fn translate_search_response( ), // A sparse search attaches each body for its column projection, so // the body stays after the filter. - PhysicalPlan::Vector(VectorOp::SparseSearch { + VectorOp::SparseSearch { collection, filters, top_k, .. - }) => translate_vector_search_payload( - payload, - state, - database_id, - tenant_id, - collection.as_str(), + } => ( + collection, VectorHitCut { rls_filters: filters.as_slice(), top_k: *top_k, strip_bodies: false, }, ), - PhysicalPlan::Vector(VectorOp::MultiVectorScoreSearch { + VectorOp::MultiVectorScoreSearch { collection, top_k, .. - }) => translate_vector_search_payload( - payload, - state, - database_id, - tenant_id, - collection.as_str(), + } => ( + collection, VectorHitCut { rls_filters: &[], top_k: *top_k, strip_bodies: false, }, ), - PhysicalPlan::Text( - TextOp::Search { collection, .. } - | TextOp::PhraseSearch { collection, .. } - | TextOp::BM25ScoreScan { collection, .. }, - ) => translate_text_search_payload( - payload, - state, - database_id, - tenant_id, - collection.as_str(), - ), - PhysicalPlan::Text(TextOp::HybridSearch { collection, .. }) - | PhysicalPlan::Text(TextOp::HybridSearchTriple { collection, .. }) => { + // Writes, maintenance and index stats answer no search hits. + VectorOp::Insert { .. } + | VectorOp::BatchInsert { .. } + | VectorOp::Delete { .. } + | VectorOp::DeleteBySurrogate { .. } + | VectorOp::SetParams { .. } + | VectorOp::DropIndex { .. } + | VectorOp::QueryStats { .. } + | VectorOp::Seal { .. } + | VectorOp::CompactIndex { .. } + | VectorOp::Rebuild { .. } + | VectorOp::SparseInsert { .. } + | VectorOp::SparseDelete { .. } + | VectorOp::MultiVectorInsert { .. } + | VectorOp::MultiVectorDelete { .. } + | VectorOp::DirectUpsert { .. } + | VectorOp::DirectInsert { .. } + | VectorOp::DirectInsertIfAbsent { .. } + | VectorOp::DirectBatchWrite { .. } + | VectorOp::DirectDelete { .. } + | VectorOp::DirectTruncate { .. } + | VectorOp::DirectUpdate { .. } + | VectorOp::ResolveDirectWrite(_) + | VectorOp::ResolvedDirectWrite { .. } => return Ok(payload.to_vec()), + }; + translate_vector_search_payload( + payload, + scope.state, + scope.database_id, + scope.tenant_id, + collection.as_str(), + cut, + ) + .await +} + +/// The text search ops' translation. Every other text op carries no hits. +async fn translate_text_op( + payload: &[u8], + op: &TextOp, + scope: TranslateScope<'_>, +) -> crate::Result> { + match op { + TextOp::Search { collection, .. } + | TextOp::PhraseSearch { collection, .. } + | TextOp::BM25ScoreScan { collection, .. } => { + translate_text_search_payload( + payload, + scope.state, + scope.database_id, + scope.tenant_id, + collection.as_str(), + ) + .await + } + TextOp::HybridSearch { collection, .. } | TextOp::HybridSearchTriple { collection, .. } => { translate_hybrid_search_payload( payload, - state, - database_id, - tenant_id, + scope.state, + scope.database_id, + scope.tenant_id, collection.as_str(), ) + .await + } + TextOp::FtsIndexDoc { .. } | TextOp::FtsDeleteDoc { .. } | TextOp::SetTextConfig { .. } => { + Ok(payload.to_vec()) + } + } +} + +/// A whole GraphRAG fusion's translation. Its coordinator stages and every +/// other graph op answer no client rows to name. +async fn translate_graph_op( + payload: &[u8], + op: &GraphOp, + scope: TranslateScope<'_>, +) -> crate::Result> { + match op { + GraphOp::RagFusion { + collection, + stage: RagStage::Local, + .. + } => { + translate_rag_fusion_payload( + payload, + scope.state, + scope.database_id, + scope.tenant_id, + collection.as_str(), + ) + .await + } + GraphOp::RagFusion { + stage: RagStage::ExportLegs | RagStage::Bindings { .. }, + .. } - _ => payload.to_vec(), + | GraphOp::EdgePut { .. } + | GraphOp::EdgePutBatch { .. } + | GraphOp::EdgeDelete { .. } + | GraphOp::EdgeDeleteBatch { .. } + | GraphOp::ResolveEdgeDelete(_) + | GraphOp::SetNodeLabels { .. } + | GraphOp::RemoveNodeLabels { .. } + | GraphOp::Hop { .. } + | GraphOp::Neighbors { .. } + | GraphOp::NeighborsMulti { .. } + | GraphOp::Path { .. } + | GraphOp::Subgraph { .. } + | GraphOp::Algo { .. } + | GraphOp::Match { .. } + | GraphOp::MatchContinuation { .. } + | GraphOp::MatchVarLenResume { .. } + | GraphOp::BspSuperstep(_) + | GraphOp::WccSuperstep(_) + | GraphOp::TemporalNeighbors { .. } + | GraphOp::TemporalAlgorithm { .. } + | GraphOp::Stats { .. } + | GraphOp::NodeEdgeGuard { .. } + | GraphOp::NodePresenceGuard { .. } + | GraphOp::TruncateEdges { .. } + | GraphOp::NodePresenceRead { .. } => Ok(payload.to_vec()), } } diff --git a/nodedb/src/control/server/response_translate/mod.rs b/nodedb/src/control/server/response_translate/mod.rs index df235a662..ce27cde59 100644 --- a/nodedb/src/control/server/response_translate/mod.rs +++ b/nodedb/src/control/server/response_translate/mod.rs @@ -5,6 +5,8 @@ pub mod dispatch; pub mod hit_key; +pub mod pk_resolve; +pub mod rag_fusion; pub mod text_hybrid; pub mod vector; diff --git a/nodedb/src/control/server/response_translate/pk_resolve.rs b/nodedb/src/control/server/response_translate/pk_resolve.rs new file mode 100644 index 000000000..d1264347d --- /dev/null +++ b/nodedb/src/control/server/response_translate/pk_resolve.rs @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Surrogate → primary-key resolution, the one naming step every search +//! response translator runs. +//! +//! A collection's bindings live on its home vShard's group. This node's +//! catalog answers first. In a cluster, every surrogate it binds no key to +//! goes to the home leader in one `SurrogateHolders` request. A surrogate +//! neither binds keeps no key, and the caller keeps the row's decimal. + +use std::collections::{HashMap, HashSet}; + +use nodedb_physical::physical_plan::ClusterEventOp; +use nodedb_types::{CollectionKey, DatabaseId, Surrogate, TenantId}; + +use crate::bridge::envelope::PhysicalPlan; +use crate::control::gateway::dispatcher::{DispatchRouteParams, dispatch_route}; +use crate::control::gateway::live_leaders::resolve_live_decision; +use crate::control::gateway::version_set::GatewayVersionSet; +use crate::control::gateway::{RouteDecision, TaskRoute}; +use crate::control::server::graph_dispatch::cluster_resolve::gateway_shared; +use crate::control::server::shared::session::StatementDeadline; +use crate::control::state::SharedState; +use crate::types::TraceId; + +/// `(collection, surrogate, pk)`: one answer of a `SurrogateHolders` request. +type Holder = (String, u32, Vec); + +/// The primary keys a set of surrogates resolved to. +#[derive(Debug, Default)] +pub(crate) struct SurrogateKeys(HashMap); + +impl SurrogateKeys { + /// The key `surrogate` resolved to. `None` when no binding names it. + pub(crate) fn get(&self, surrogate: Surrogate) -> Option<&str> { + self.0.get(&surrogate).map(String::as_str) + } +} + +/// The client identity of a row bound to `pk_bytes`. A headless row binds +/// the big-endian bytes of its own surrogate and carries the decimal +/// identity a minted row carries. Any other key is its UTF-8 text. +pub(crate) fn pk_identity(pk_bytes: &[u8], surrogate: Surrogate) -> Option { + if pk_bytes == surrogate.as_u32().to_be_bytes() { + return Some(surrogate.as_u32().to_string()); + } + std::str::from_utf8(pk_bytes).ok().map(str::to_owned) +} + +/// Resolve every surrogate of `surrogates` to its primary key in +/// `collection`, the plan's database-qualified name. A name that does not +/// de-qualify resolves nothing. A catalog read error or a refused home +/// request fails the call: a row never answers a key nobody vouched for. +pub(crate) async fn resolve_surrogate_pks( + state: &SharedState, + database_id: DatabaseId, + tenant_id: TenantId, + collection: &str, + surrogates: impl IntoIterator, +) -> crate::Result { + let mut keys = SurrogateKeys::default(); + let Ok(key) = CollectionKey::from_qualified_str(database_id, collection) else { + return Ok(keys); + }; + let mut seen: HashSet = HashSet::new(); + let wanted: Vec = surrogates + .into_iter() + .filter(|surrogate| seen.insert(*surrogate)) + .collect(); + // Every surrogate of the response resolves in one catalog read. + let bound = state + .credentials + .catalog() + .get_pks_for_surrogates(key, tenant_id, &wanted)?; + let mut misses: Vec = Vec::new(); + for (surrogate, pk) in wanted.into_iter().zip(bound) { + match pk.and_then(|pk| pk_identity(&pk, surrogate)) { + Some(identity) => { + keys.0.insert(surrogate, identity); + } + None => misses.push(surrogate.as_u32()), + } + } + if misses.is_empty() { + return Ok(keys); + } + for (_, raw, pk) in home_holders(state, database_id, tenant_id, key, misses).await? { + let surrogate = Surrogate::new(raw); + if let Some(identity) = pk_identity(&pk, surrogate) { + keys.0.insert(surrogate, identity); + } + } + Ok(keys) +} + +/// The keys the collection home leader binds `surrogates` to. Empty on a +/// single node and when this node leads the home: its catalog answered. +async fn home_holders( + state: &SharedState, + database_id: DatabaseId, + tenant_id: TenantId, + key: CollectionKey<'_>, + surrogates: Vec, +) -> crate::Result> { + if state.cluster_routing.is_none() { + return Ok(Vec::new()); + } + let home = key.vshard().as_u32(); + let decision = resolve_live_decision(state, home); + if matches!(decision, RouteDecision::Local) { + return Ok(Vec::new()); + } + let bare = key.name(); + let plan = PhysicalPlan::ClusterEvent(ClusterEventOp::SurrogateHolders { + tenant_id: tenant_id.as_u64(), + database_id, + entries: surrogates + .into_iter() + .map(|surrogate| (bare.to_owned(), surrogate)) + .collect(), + }); + let shared = gateway_shared(state)?; + let outcome = dispatch_route(DispatchRouteParams { + route: TaskRoute { + plan, + decision, + vshard_id: home, + }, + shared: &shared, + tenant_id, + database_id, + trace_id: TraceId::ZERO, + deadline_ms: StatementDeadline::of_statement(state).hop_budget_ms(), + version_set: &GatewayVersionSet::from_pairs(Vec::new()), + txn_id: None, + linearizable: false, + user_roles: &[], + }) + .await?; + let mut holders = Vec::new(); + for payload in outcome + .payloads + .iter() + .filter(|payload| !payload.is_empty()) + { + let mut answer: Vec = + zerompk::from_msgpack(payload).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!( + "surrogate holders of '{}' from its home leader do not decode: {e}", + key.name() + ), + })?; + holders.append(&mut answer); + } + Ok(holders) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn self_keyed_binding_resolves_to_decimal() { + let surrogate = Surrogate::new(7); + assert_eq!( + pk_identity(&7u32.to_be_bytes(), surrogate).as_deref(), + Some("7") + ); + } + + #[test] + fn a_text_key_resolves_to_itself() { + assert_eq!( + pk_identity(b"lonely", Surrogate::new(7)).as_deref(), + Some("lonely") + ); + } + + #[test] + fn a_non_utf8_key_resolves_to_nothing() { + assert_eq!(pk_identity(&[0xff, 0xfe], Surrogate::new(7)), None); + } +} diff --git a/nodedb/src/control/server/response_translate/rag_fusion.rs b/nodedb/src/control/server/response_translate/rag_fusion.rs new file mode 100644 index 000000000..d1e2fbd08 --- /dev/null +++ b/nodedb/src/control/server/response_translate/rag_fusion.rs @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Surrogate → user-PK naming for GraphRAG fusion responses. +//! +//! The fusion answers a row named by no graph node with its decimal and its +//! surrogate. This translator names that row by its primary key, through +//! [`super::pk_resolve::resolve_surrogate_pks`], the resolver every search +//! translator runs, and clears the surrogate. Naming runs after fusion, so +//! scores and ranks are the fusion's own. In a cluster it runs on the +//! coordinator, and the resolver asks the collection home leader for every +//! key this node's catalog lacks. + +use nodedb_types::{DatabaseId, Surrogate, TenantId}; + +use crate::control::state::SharedState; +use crate::data::executor::response_codec::GraphRagResponse; + +use super::pk_resolve::resolve_surrogate_pks; + +/// Decode a fusion payload, name its rows, and re-encode it. `collection` is +/// the plan's database-qualified name. A row no binding names keeps its +/// decimal. A payload that does not decode fails the call. +pub async fn translate_rag_fusion_payload( + payload: &[u8], + state: &SharedState, + database_id: DatabaseId, + tenant_id: TenantId, + collection: &str, +) -> crate::Result> { + if payload.is_empty() { + return Ok(Vec::new()); + } + let mut response: GraphRagResponse = + zerompk::from_msgpack(payload).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("graph rag fusion response does not decode: {e}"), + })?; + let unnamed: Vec = response + .results + .iter() + .filter_map(|row| row.surrogate.map(Surrogate::new)) + .collect(); + if unnamed.is_empty() { + return Ok(payload.to_vec()); + } + let keys = resolve_surrogate_pks(state, database_id, tenant_id, collection, unnamed).await?; + for row in &mut response.results { + if let Some(raw) = row.surrogate.take() + && let Some(key) = keys.get(Surrogate::new(raw)) + { + row.node_id = key.to_owned(); + } + } + zerompk::to_msgpack_vec(&response).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("graph rag fusion response does not encode: {e}"), + }) +} diff --git a/nodedb/src/control/server/response_translate/text_hybrid.rs b/nodedb/src/control/server/response_translate/text_hybrid.rs index f327bc187..5aec86d1b 100644 --- a/nodedb/src/control/server/response_translate/text_hybrid.rs +++ b/nodedb/src/control/server/response_translate/text_hybrid.rs @@ -17,17 +17,17 @@ //! `__local_` sentinel for a vector-leg hit with no surrogate binding). //! This is the genuine gap: without this translator `SELECT id` against a //! hybrid query has no `id` field to read at all. Both paths resolve through -//! the same [`super::vector::resolve_surrogate_pk`] catalog call the vector +//! [`super::pk_resolve::resolve_surrogate_pks`], the resolver the vector //! translator uses. -use nodedb_types::{DatabaseId, TenantId}; +use nodedb_types::{DatabaseId, Surrogate, TenantId}; use serde_json::Value as JsonValue; use crate::control::state::SharedState; use crate::data::executor::response_codec::decode_payload_to_json; use super::hit_key::parse_surrogate_hex; -use super::vector::resolve_surrogate_pk; +use super::pk_resolve::resolve_surrogate_pks; /// Decode the DP-side JSON/msgpack array of `TextOp::Search` / /// `PhraseSearch` / `BM25ScoreScan` rows (`{id: , data: {...}}`), and for @@ -36,33 +36,34 @@ use super::vector::resolve_surrogate_pk; /// under that column. A declared-key row never gains an `id` beside its key. /// Rows that hold the column (the common case) are left untouched. An /// unresolved or headless surrogate is left untouched (no fabricated PK). -/// On any decode failure, or a catalog error resolving the identity column, +/// On a decode failure, or a catalog error resolving the identity column, /// the payload is returned unchanged: a row never gains a wrongly named -/// column. -pub fn translate_text_search_payload( +/// column. A resolver error fails the call. +pub async fn translate_text_search_payload( payload: &[u8], state: &SharedState, database_id: DatabaseId, tenant_id: TenantId, collection: &str, -) -> Vec { +) -> crate::Result> { if payload.is_empty() { - return payload.to_vec(); + return Ok(payload.to_vec()); } let text = decode_payload_to_json(payload); let Ok(JsonValue::Array(mut rows)) = sonic_rs::from_str::(&text) else { - return payload.to_vec(); + return Ok(payload.to_vec()); }; let Ok(identity_column) = identity_column(state, database_id, tenant_id, collection) else { - return payload.to_vec(); + return Ok(payload.to_vec()); }; - for row in &mut rows { + let mut unnamed: Vec<(usize, Surrogate)> = Vec::new(); + for (index, row) in rows.iter().enumerate() { let JsonValue::Object(map) = row else { continue; }; - let Some(JsonValue::String(hex_id)) = map.get("id").cloned() else { + let Some(JsonValue::String(hex_id)) = map.get("id") else { continue; }; let data_has_identity = matches!( @@ -72,20 +73,34 @@ pub fn translate_text_search_payload( if data_has_identity { continue; } - let Some(surrogate) = parse_surrogate_hex(&hex_id) else { - continue; - }; - if let Some(pk) = resolve_surrogate_pk(state, database_id, tenant_id, collection, surrogate) + if let Some(surrogate) = parse_surrogate_hex(hex_id) { + unnamed.push((index, surrogate)); + } + } + if unnamed.is_empty() { + return Ok(payload.to_vec()); + } + let keys = resolve_surrogate_pks( + state, + database_id, + tenant_id, + collection, + unnamed.iter().map(|(_, surrogate)| *surrogate), + ) + .await?; + for (index, surrogate) in &unnamed { + if let Some(key) = keys.get(*surrogate) + && let Some(JsonValue::Object(map)) = rows.get_mut(*index) && let Some(JsonValue::Object(inner)) = map.get_mut("data") { - inner.insert(identity_column.clone(), JsonValue::String(pk)); + inner.insert(identity_column.clone(), JsonValue::String(key.to_owned())); } } - match sonic_rs::to_string(&JsonValue::Array(rows)) { + Ok(match sonic_rs::to_string(&JsonValue::Array(rows)) { Ok(s) => s.into_bytes(), Err(_) => payload.to_vec(), - } + }) } /// The column `collection`'s rows render their identity under: its declared @@ -120,41 +135,59 @@ fn identity_column( /// so `doc_id` itself is rewritten to the row's identity too: the catalog PK /// when one is declared, else the surrogate's decimal string. A `__local_` /// sentinel (no surrogate binding) passes through untouched in both fields. -/// On any decode failure the payload is returned unchanged. -pub fn translate_hybrid_search_payload( +/// On a decode failure the payload is returned unchanged. A resolver error +/// fails the call. +pub async fn translate_hybrid_search_payload( payload: &[u8], state: &SharedState, database_id: DatabaseId, tenant_id: TenantId, collection: &str, -) -> Vec { +) -> crate::Result> { if payload.is_empty() { - return payload.to_vec(); + return Ok(payload.to_vec()); } let text = decode_payload_to_json(payload); let Ok(JsonValue::Array(mut rows)) = sonic_rs::from_str::(&text) else { - return payload.to_vec(); + return Ok(payload.to_vec()); }; - for row in &mut rows { - let JsonValue::Object(map) = row else { - continue; - }; - let Some(JsonValue::String(hex_id)) = map.get("doc_id").cloned() else { + let bound: Vec<(usize, Surrogate)> = rows + .iter() + .enumerate() + .filter_map(|(index, row)| match row { + JsonValue::Object(map) => match map.get("doc_id") { + Some(JsonValue::String(hex_id)) => { + parse_surrogate_hex(hex_id).map(|surrogate| (index, surrogate)) + } + _ => None, + }, + _ => None, + }) + .collect(); + let keys = resolve_surrogate_pks( + state, + database_id, + tenant_id, + collection, + bound.iter().map(|(_, surrogate)| *surrogate), + ) + .await?; + for (index, surrogate) in &bound { + let Some(JsonValue::Object(map)) = rows.get_mut(*index) else { continue; }; - let Some(surrogate) = parse_surrogate_hex(&hex_id) else { - continue; - }; - let identity = resolve_surrogate_pk(state, database_id, tenant_id, collection, surrogate) + let identity = keys + .get(*surrogate) + .map(str::to_owned) .unwrap_or_else(|| surrogate.as_u32().to_string()); map.insert("id".to_string(), JsonValue::String(identity.clone())); map.insert("doc_id".to_string(), JsonValue::String(identity)); } - match sonic_rs::to_string(&JsonValue::Array(rows)) { + Ok(match sonic_rs::to_string(&JsonValue::Array(rows)) { Ok(s) => s.into_bytes(), Err(_) => payload.to_vec(), - } + }) } diff --git a/nodedb/src/control/server/response_translate/vector.rs b/nodedb/src/control/server/response_translate/vector.rs index 02d73e944..e4009f8a0 100644 --- a/nodedb/src/control/server/response_translate/vector.rs +++ b/nodedb/src/control/server/response_translate/vector.rs @@ -16,9 +16,9 @@ //! - decode failures are non-fatal — the original payload is returned //! so the client still sees the raw search hits. //! -//! [`resolve_surrogate_pk`] is the single catalog lookup shared by this -//! translator and the full-text / hybrid translators in `text_hybrid.rs` — -//! see `dispatch.rs` for the plan-to-translator routing. +//! A hit names its row through [`super::pk_resolve::resolve_surrogate_pks`], +//! the one resolver every search translator runs. See `dispatch.rs` for the +//! plan-to-translator routing. use nodedb_types::DatabaseId; use nodedb_types::Surrogate; @@ -26,6 +26,7 @@ use nodedb_types::TenantId; use serde::{Deserialize, Serialize}; use super::hit_key::parse_surrogate_hex; +use super::pk_resolve::resolve_surrogate_pks; use crate::bridge::scan_filter::ScanFilter; use crate::control::state::SharedState; @@ -130,72 +131,77 @@ pub(crate) fn filter_vector_hits( }) } -/// Resolve one surrogate to its user-facing primary key via the catalog. -/// -/// Shared by the vector, full-text, and hybrid (RRF) response translators — -/// every search-hit path resolves the internal `u32` surrogate back to the -/// user's PK through this single catalog call. Returns `None` when the -/// catalog has no PK mapping for the surrogate (headless row, or a document -/// that was never written) — callers must leave the row's identifier -/// untouched in that case rather than fabricate a value. `collection` is the -/// plan's database-qualified name, de-qualified into the canonical key; a -/// name that does not de-qualify also resolves to `None`. -pub(crate) fn resolve_surrogate_pk( - state: &SharedState, - database_id: DatabaseId, - tenant_id: TenantId, - collection: &str, - surrogate: Surrogate, -) -> Option { - let catalog = state.credentials.catalog(); - let key = nodedb_types::CollectionKey::from_qualified_str(database_id, collection).ok()?; - let pk_bytes = catalog - .get_pk_for_surrogate(key, tenant_id, surrogate) - .ok()??; - String::from_utf8(pk_bytes).ok() -} - -/// Decode the DP-side msgpack array of `VectorSearchHit`, fill each -/// row's `doc_id` from the catalog using `id` as the surrogate, and -/// re-encode. On any decode failure the payload is returned unchanged. -pub fn translate_vector_search_payload( +/// Decode the DP-side msgpack array of `VectorSearchHit`, flatten each hit +/// into a row, and name each row whose body carries no `id` by its primary +/// key. A payload that is not a hit array returns unchanged. +pub async fn translate_vector_search_payload( payload: &[u8], state: &SharedState, database_id: DatabaseId, tenant_id: TenantId, collection: &str, cut: VectorHitCut<'_>, -) -> Vec { +) -> crate::Result> { if payload.is_empty() { - return payload.to_vec(); + return Ok(payload.to_vec()); } - let first = payload[0]; - if first == b'[' || first == b'{' || first == b'"' { - return payload.to_vec(); + // A payload that opens with no msgpack array marker carries no hit + // array: JSON text starts with `[`, `{` or `"`, and a msgpack scalar or + // map has its own marker. It passes through. A msgpack array that does + // not decode as hits is an error. + if !is_msgpack_array(payload[0]) { + return Ok(payload.to_vec()); } - let mut hits: Vec = match zerompk::from_msgpack(payload) { - Ok(h) => h, - Err(_) => return payload.to_vec(), - }; + let mut hits: Vec = + zerompk::from_msgpack(payload).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("vector search hits do not decode: {e}"), + })?; apply_rls_filter(&mut hits, cut.rls_filters, cut.top_k, cut.strip_bodies); - for hit in &mut hits { - if hit.doc_id.is_some() { - continue; - } - // A headless hit (no surrogate binding) has no PK to resolve; leave - // it untouched exactly as `text_hybrid.rs` does for the same sentinel. - let Some(surrogate) = parse_surrogate_hex(&hit.id) else { - continue; - }; - if let Some(pk) = resolve_surrogate_pk(state, database_id, tenant_id, collection, surrogate) - { - hit.doc_id = Some(pk); + let FlattenedHits { mut rows, unnamed } = flatten_hits(&hits)?; + let keys = resolve_surrogate_pks( + state, + database_id, + tenant_id, + collection, + unnamed.iter().map(|(_, surrogate)| *surrogate), + ) + .await?; + for (index, surrogate) in &unnamed { + if let (Some(key), Some(row)) = (keys.get(*surrogate), rows.get_mut(*index)) { + row.insert("id".into(), serde_json::json!(key)); } } + sonic_rs::to_string(&rows) + .map(String::into_bytes) + .map_err(|e| crate::Error::Serialization { + format: "json".into(), + detail: format!("vector search rows do not encode: {e}"), + }) +} + +/// Whether `marker` opens a msgpack array: fixarray, array 16 or array 32. +fn is_msgpack_array(marker: u8) -> bool { + matches!(marker, 0x90..=0x9f | 0xdc | 0xdd) +} + +/// One flattened hit row, keyed by column. +type HitRow = std::collections::BTreeMap; + +/// The rows `flatten_hits` builds, and the rows the caller names. +struct FlattenedHits { + rows: Vec, + /// The index and surrogate of each row that carries no `id` of its own. + unnamed: Vec<(usize, Surrogate)>, +} +/// Flatten `hits` into rows. Also returns the index and surrogate of each +/// row that carries no `id` of its own, for the caller to name. A stored +/// body that does not decode fails the call. +fn flatten_hits(hits: &[Hit]) -> crate::Result { // Slow-path column projection: when `body` is a msgpack-encoded payload // map, decode it and surface fields alongside `id` / `distance` so client // SQL projections like `SELECT id, label, vector_distance(...)` see the @@ -210,45 +216,55 @@ pub fn translate_vector_search_payload( // Decoding with the tagged `Value` codec here instead would read exactly // one of the two collection kinds and silently drop the columns of the // other. - use std::collections::BTreeMap; - let flattened: Vec> = hits - .iter() - .map(|h| { - let mut obj: BTreeMap = BTreeMap::new(); - obj.insert("distance".into(), serde_json::json!(h.distance)); - // Flatten body fields first so the document's own `id` field wins - // over the internal surrogate integer. Body fields are only present - // when the Data Plane performed the slow-path fetch. - if let Some(ref body) = h.body - && let Ok(serde_json::Value::Object(map)) = nodedb_types::json_from_msgpack(body) - { + let mut rows: Vec = Vec::with_capacity(hits.len()); + let mut unnamed: Vec<(usize, Surrogate)> = Vec::new(); + for h in hits { + let mut obj = HitRow::new(); + obj.insert("distance".into(), serde_json::json!(h.distance)); + // Body fields go first, so the document's own `id` field wins over + // the surrogate. Only a slow-path fetch attaches a body. A stored + // body that does not decode is corruption and fails the response. + if let Some(ref body) = h.body { + let decoded = + nodedb_types::json_from_msgpack(body).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!( + "vector hit {} carries a stored body that does not decode: {e}", + h.id + ), + })?; + if let serde_json::Value::Object(map) = decoded { for (k, v) in map { obj.insert(k, v); } } - // A storage key never reaches a client: a bound hit's client - // identity is its surrogate's decimal string; a headless hit has - // no surrogate, so the sentinel is the only identity it carries. - let identity = match parse_surrogate_hex(&h.id) { - Some(surrogate) => surrogate.as_u32().to_string(), - None => h.id.clone(), - }; - // Fall back to catalog-resolved doc_id or the decimal identity - // when the body didn't provide an "id" field (e.g. skip_payload_fetch). - if !obj.contains_key("id") { - if let Some(ref doc) = h.doc_id { + } + // A storage key never reaches a client: a bound hit's identity is its + // surrogate's decimal string. A headless hit carries only the sentinel. + let surrogate = parse_surrogate_hex(&h.id); + let identity = match surrogate { + Some(surrogate) => surrogate.as_u32().to_string(), + None => h.id.clone(), + }; + // A row with no `id` of its own takes its key, named by the caller, + // and keeps its decimal identity when no binding names it. + if !obj.contains_key("id") { + match (&h.doc_id, surrogate) { + (Some(doc), _) => { obj.insert("id".into(), serde_json::json!(doc)); - } else { + } + (None, Some(surrogate)) => { + unnamed.push((rows.len(), surrogate)); + obj.insert("id".into(), serde_json::json!(identity)); + } + (None, None) => { obj.insert("id".into(), serde_json::json!(identity)); } } - // Always expose internal surrogate for debugging / join use. - obj.insert("_surrogate".into(), serde_json::json!(identity)); - obj - }) - .collect(); - if let Ok(s) = sonic_rs::to_string(&flattened) { - return s.into_bytes(); + } + // The surrogate is exposed for join use. + obj.insert("_surrogate".into(), serde_json::json!(identity)); + rows.push(obj); } - zerompk::to_msgpack_vec(&hits).unwrap_or_else(|_| payload.to_vec()) + Ok(FlattenedHits { rows, unnamed }) } diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch_write.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch_write.rs index 64017da79..4e5a03fe2 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch_write.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch_write.rs @@ -11,7 +11,7 @@ use crate::control::security::audit::ArcAuditEmitter; use crate::control::security::identity::{AuthenticatedIdentity, Permission}; use crate::control::sequence::SessionSequenceAccess; use crate::control::server::pgwire::types::error_to_sqlstate; -use crate::control::server::response_shape::compose::{ShapeOutcome, shape_response_materialized}; +use crate::control::server::response_shape::compose::{ShapeOutcome, shape_returning_materialized}; use crate::control::server::response_shape::redaction::QueryRedaction; use crate::control::server::response_shape::request::MaterializedShapeRequest; use crate::control::server::response_shape::schema::OutputSchema; @@ -241,7 +241,7 @@ impl ReturningShape<'_> { self.database_id, tenant_id, ); - let outcome = shape_response_materialized(MaterializedShapeRequest { + let outcome = shape_returning_materialized(MaterializedShapeRequest { payload, plan, plan_kind: PlanKind::ReturningRows, diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs index 7cfdcb4d9..8c68e4617 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs @@ -166,11 +166,19 @@ pub async fn convert_collection( // The declared-field record follows the converted columns, as // `CREATE COLLECTION` records them. coll.fields = converted_fields(&columns, &generated_sources); - let schema = nodedb_types::columnar::StrictSchema { - columns, - version: 1, - dropped_columns: Vec::new(), - bitemporal: false, + // A bitemporal collection keeps its flag across the CONVERT, so + // its strict schema carries the reserved version columns the Data + // Plane encoded every version with. + let schema = if coll.bitemporal { + nodedb_types::columnar::StrictSchema::new_bitemporal(columns) + .map_err(|e| err("42P16", format!("bitemporal strict schema: {e}")))? + } else { + nodedb_types::columnar::StrictSchema { + columns, + version: 1, + dropped_columns: Vec::new(), + bitemporal: false, + } }; if target_type == "kv" { nodedb_types::CollectionType::kv(schema) diff --git a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs index 779564f12..aee3cbf1f 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/graph_ops/rag_fusion.rs @@ -15,6 +15,7 @@ use nodedb_sql::ddl_ast::GraphDirection; use crate::bridge::envelope::PhysicalPlan; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::response_shape::types::ShapedRows; +use crate::control::server::response_translate::translate_search_response; use crate::control::server::shared::ddl::user_dispatch; use crate::control::server::shared::session::StatementDeadline; use crate::control::state::SharedState; @@ -125,7 +126,8 @@ pub async fn rag_fusion( identity, database_id, collection: &collection, - plan, + // The response translation below reads the plan the fusion ran. + plan: plan.clone(), deadline: StatementDeadline::of_statement(state), // Admitted at this request's own transport entry; this handler has no // session or peer information of its own. @@ -135,7 +137,12 @@ pub async fn rag_fusion( .await .map_err(|e| DdlError::from_error(&e))?; - let json_text = response_codec::decode_payload_to_json(&payload); + // Rows named by no graph node take their primary keys here, through the + // resolver every search response uses. + let named = translate_search_response(&payload, &plan, state, database_id, identity.tenant_id) + .await + .map_err(|e| DdlError::from_error(&e))?; + let json_text = response_codec::decode_payload_to_json(&named); let mut row = Map::new(); row.insert("result".to_string(), JsonValue::String(json_text)); diff --git a/nodedb/src/control/server/shared/session/expander_stage.rs b/nodedb/src/control/server/shared/session/expander_stage.rs index 8fd44159d..ef123ae71 100644 --- a/nodedb/src/control/server/shared/session/expander_stage.rs +++ b/nodedb/src/control/server/shared/session/expander_stage.rs @@ -251,6 +251,7 @@ where state, &op, sessions.tx_id(session_id), + None, ) .await .map_err(StagingGateError::Dispatch)?; diff --git a/nodedb/src/control/server/shared/session/mod.rs b/nodedb/src/control/server/shared/session/mod.rs index df8085fda..d3bbbf8f2 100644 --- a/nodedb/src/control/server/shared/session/mod.rs +++ b/nodedb/src/control/server/shared/session/mod.rs @@ -32,6 +32,7 @@ pub mod overlay_drop; mod own_writes; mod params; pub mod pending_shard_reads; +mod predicted_stage; pub mod read_consistency; pub mod read_set; pub mod record_reads; diff --git a/nodedb/src/control/server/shared/session/node_delete_stage.rs b/nodedb/src/control/server/shared/session/node_delete_stage.rs index ad021e111..14cae68e5 100644 --- a/nodedb/src/control/server/shared/session/node_delete_stage.rs +++ b/nodedb/src/control/server/shared/session/node_delete_stage.rs @@ -1,12 +1,13 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Staging of a transaction delete's node-delete edge tasks. +//! Staging of the edge tasks a transaction write derives. //! //! A delete on an edge-bearing collection inside a transaction block carries -//! the edge tasks `txn_node_delete_tasks` derives. Each `EdgeDelete` stages -//! into the transaction's graph overlay now, so later reads in the -//! transaction see the edge gone. Each `NodeEdgeGuard` is buffered, and -//! COMMIT checks it. +//! the edge tasks `txn_node_delete_tasks` derives. An update on one carries +//! its implicit-edge reconciliation. Each `EdgeDelete` and `EdgePut` stages +//! into the transaction's graph overlay at the statement, so later reads in +//! the transaction see it. Each `NodeEdgeGuard` is buffered, and COMMIT +//! checks it. use std::future::Future; diff --git a/nodedb/src/control/server/shared/session/predicted_stage.rs b/nodedb/src/control/server/shared/session/predicted_stage.rs new file mode 100644 index 000000000..244bf9082 --- /dev/null +++ b/nodedb/src/control/server/shared/session/predicted_stage.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Staging of one in-transaction write with its prediction and the edge +//! tasks it derives. +//! +//! A predicate write reads its rows first (`predict_txn_bulk_write`). A +//! delete on an edge-bearing collection derives its node-delete edge tasks +//! from that same read. An update on one derives its implicit-edge +//! reconciliation from it. The stage refuses with `OllpRetryRequired` when +//! its rows no longer match the read, staging nothing, and the read runs +//! again. Past the retry budget the statement fails with a serialization +//! conflict the client retries. + +use std::future::Future; + +use crate::bridge::envelope::{ErrorCode, Response}; +use crate::control::planner::calvin::node_delete_txn::txn_node_delete_tasks; +use crate::control::planner::calvin::retry_budget::OllpRetryBudget; +use crate::control::planner::calvin::txn_predict::predict_txn_bulk_write; +use crate::control::server::shared::sql::staging_predicates::is_stageable_write; +use crate::control::state::SharedState; +use nodedb_physical::physical_task::PhysicalTask; + +use super::connection::SessionId; +use super::node_delete_stage::stage_node_delete_tasks; +use super::staging_gate::{InTxnRoute, StagingGateError, stage_write}; +use super::store::SessionStore; +use super::txn_expand::expand_for_buffering; + +/// Stage or buffer `task` in the session's transaction, with its prediction +/// and the edge tasks it derives. +pub(super) async fn stage_predicted_write( + state: &SharedState, + sessions: &SessionStore, + session_id: SessionId, + task: PhysicalTask, + dispatch: &F, +) -> Result +where + F: Fn(PhysicalTask) -> Fut, + Fut: Future>, +{ + let txn_id = sessions.tx_id(session_id); + // The shared OLLP retry policy, opened on the first refusal: the + // orchestrator's budget and backoff. A node with no orchestrator retries + // nothing. + let mut budget: Option> = None; + loop { + let mut task = task.clone(); + let read = predict_txn_bulk_write(state, &mut task, txn_id) + .await + .map_err(StagingGateError::Dispatch)?; + // A delete on an edge-bearing collection carries the edge tasks of + // each row it deletes, read before the delete stages. + let mut derived = + txn_node_delete_tasks(state, &task, txn_id, read.as_ref().map(|read| &read.scan)) + .await + .map_err(StagingGateError::Dispatch)?; + let predicate_class = read.as_ref().map(|read| read.predicate_class); + if let Some(read) = read { + derived.extend(read.edge_tasks); + } + let route = if is_stageable_write(&task.plan) { + let staged = stage_write(state, sessions, session_id, task, dispatch).await; + match (staged, predicate_class) { + (Ok(outcome), _) => InTxnRoute::Staged(outcome), + ( + Err(StagingGateError::Rejected { + code: Some(ErrorCode::OllpRetryRequired), + }), + Some(predicate_class), + ) => { + if budget.is_none() { + budget = state.ollp_orchestrator.get().map(|orchestrator| { + OllpRetryBudget::configured(orchestrator, predicate_class) + }); + } + match budget.as_mut() { + Some(budget) if !budget.exhausted() => { + budget.spend().await; + continue; + } + // The statement's earlier reads stand, so the + // transaction fails with a conflict the client + // retries. + Some(_) | None => { + return Err(StagingGateError::Dispatch( + crate::Error::CalvinSerializationConflict, + )); + } + } + } + (Err(error), _) => return Err(error), + } + } else { + for shard_task in expand_for_buffering(task).map_err(StagingGateError::Dispatch)? { + sessions.buffer_write(session_id, shard_task); + } + InTxnRoute::Buffered + }; + stage_node_delete_tasks(state, sessions, session_id, derived, dispatch).await?; + return Ok(route); + } +} diff --git a/nodedb/src/control/server/shared/session/read_set.rs b/nodedb/src/control/server/shared/session/read_set.rs index 69d06f88c..299f24a3a 100644 --- a/nodedb/src/control/server/shared/session/read_set.rs +++ b/nodedb/src/control/server/shared/session/read_set.rs @@ -780,6 +780,7 @@ mod tests { projection: Vec::new(), limit: 0, offset: 0, + system_as_of_ms: None, }) } diff --git a/nodedb/src/control/server/shared/session/staging_gate.rs b/nodedb/src/control/server/shared/session/staging_gate.rs index e4f480743..cec9b54c1 100644 --- a/nodedb/src/control/server/shared/session/staging_gate.rs +++ b/nodedb/src/control/server/shared/session/staging_gate.rs @@ -30,7 +30,7 @@ use crate::control::server::shared::metering::{PlanMeteringInfo, meter_staged_wr use crate::control::server::shared::quota_admission::admit_quota_for_dispatch; use crate::control::server::shared::session::StatementDeadline; use crate::control::server::shared::sql::staging_predicates::{ - is_stageable_write, require_affected_count, stageable_write_shape, stages_returning, + require_affected_count, stageable_write_shape, stages_returning, }; use crate::control::server::shared::write_admission::{plan_is_write, plan_requires_txn_buffering}; use crate::control::state::SharedState; @@ -43,7 +43,6 @@ use super::connection::SessionId; use super::leader_forward::{forward_to_leader, resolve_leader}; use super::state::TransactionState; use super::store::SessionStore; -use super::txn_expand::expand_for_buffering; pub use crate::control::server::shared::sql::staging_predicates::StagedTagKind; /// Outcome of routing a single task through the in-transaction staging gate. @@ -232,40 +231,14 @@ where )); } - // A delete on an edge-bearing collection carries the edge tasks of each - // row it deletes. They are read before the delete stages, so the read - // still sees the rows. - let node_delete_tasks = - crate::control::planner::calvin::node_delete_txn::txn_node_delete_tasks( - state, - &task, - sessions.tx_id(session_id), - ) - .await - .map_err(StagingGateError::Dispatch)?; - // Point writes execute at STATEMENT time via the staging overlay (real // tag + statement-time constraint errors); the plan is still buffered so // COMMIT stays the sole durable apply. Other writes keep buffer + "OK", - // reshaped first into the per-shard plans COMMIT can replay. - let route = if is_stageable_write(&task.plan) { - InTxnRoute::Staged(stage_write(state, sessions, session_id, task, &dispatch).await?) - } else { - let buffered = expand_for_buffering(task).map_err(StagingGateError::Dispatch)?; - for shard_task in buffered { - sessions.buffer_write(session_id, shard_task); - } - InTxnRoute::Buffered - }; - super::node_delete_stage::stage_node_delete_tasks( - state, - sessions, - session_id, - node_delete_tasks, - &dispatch, - ) - .await?; - Ok(route) + // reshaped first into the per-shard plans COMMIT can replay. A predicate + // write stages with its prediction, and every write with the edge tasks + // it derives. + super::predicted_stage::stage_predicted_write(state, sessions, session_id, task, &dispatch) + .await } /// Wrap `plan` in a `MetaOp::StageWrite` task addressed to `vshard_id`, diff --git a/nodedb/src/control/server/shared/sql/staging_predicates.rs b/nodedb/src/control/server/shared/sql/staging_predicates.rs index 57f68dbda..c1bca5d87 100644 --- a/nodedb/src/control/server/shared/sql/staging_predicates.rs +++ b/nodedb/src/control/server/shared/sql/staging_predicates.rs @@ -262,13 +262,9 @@ pub fn stages_returning(plan: &PhysicalPlan) -> bool { .. } | DocumentOp::BulkUpdate { returning: Some(_), - ollp_predicted_surrogates: None, - ollp_predicted_edges: None, .. } | DocumentOp::BulkDelete { returning: Some(_), - ollp_predicted_surrogates: None, - ollp_predicted_edges: None, .. } ) diff --git a/nodedb/src/control/server/shared/statement_exec/answer.rs b/nodedb/src/control/server/shared/statement_exec/answer.rs index e504d1a40..f0b9b6829 100644 --- a/nodedb/src/control/server/shared/statement_exec/answer.rs +++ b/nodedb/src/control/server/shared/statement_exec/answer.rs @@ -78,7 +78,7 @@ impl StatementFold { /// Shape `payload`, the answer `plan` gave as `plan_kind`, and fold its /// rows. Returns the row count, `None` when the payload carries no rows. - pub(crate) fn fold_rows( + pub(crate) async fn fold_rows( &mut self, exec: &StatementExec<'_>, plan: &PhysicalPlan, @@ -97,7 +97,9 @@ impl StatementFold { tenant_id: exec.tenant_id(), redaction: Some(redaction.ctx(&exec.state.redaction)), sequences: Some(exec.sequences), - }) { + }) + .await + { Ok(ShapeOutcome::Rows(shaped)) => Ok(Some(self.fold_shaped(shaped))), Ok(ShapeOutcome::Passthrough) => Ok(None), Err(error) => Err(StatementError::Shape(error)), @@ -107,7 +109,7 @@ impl StatementFold { /// Fold a write its transaction staged. A write that answers `RETURNING` /// folds its rows in place of a count, as an autocommit one does. A /// computed-value payload (`RawPayload`) is shaped instead of counted. - pub(crate) fn fold_staged( + pub(crate) async fn fold_staged( &mut self, exec: &StatementExec<'_>, plan: &PhysicalPlan, @@ -115,7 +117,8 @@ impl StatementFold { ) -> Result<(), StatementError> { if !outcome.returning_rows.is_empty() { for rows in &outcome.returning_rows { - self.fold_rows(exec, plan, PlanKind::ReturningRows, rows)?; + self.fold_rows(exec, plan, PlanKind::ReturningRows, rows) + .await?; } return Ok(()); } @@ -128,7 +131,8 @@ impl StatementFold { } // The value rides in the payload, never as a count. if self - .fold_rows(exec, plan, describe_plan(plan), &outcome.payload)? + .fold_rows(exec, plan, describe_plan(plan), &outcome.payload) + .await? .is_none() { self.tag.fold_opaque(); @@ -139,7 +143,7 @@ impl StatementFold { /// Fold a dispatched task's answer `payload`: its rows, or its count. /// Returns the task's own row count for metering, `None` when it carries /// none. - pub(crate) fn fold_answer( + pub(crate) async fn fold_answer( &mut self, exec: &StatementExec<'_>, plan: &PhysicalPlan, @@ -154,7 +158,7 @@ impl StatementFold { self.tag.fold_opaque(); return Ok(None); } - if let Some(rows) = self.fold_rows(exec, plan, plan_kind, payload)? { + if let Some(rows) = self.fold_rows(exec, plan, plan_kind, payload).await? { return Ok(Some(rows)); } // A count-bearing write reports the rows it touched and its verb; an diff --git a/nodedb/src/control/server/shared/statement_exec/task_loop.rs b/nodedb/src/control/server/shared/statement_exec/task_loop.rs index 0abfd8ae8..9e34b0842 100644 --- a/nodedb/src/control/server/shared/statement_exec/task_loop.rs +++ b/nodedb/src/control/server/shared/statement_exec/task_loop.rs @@ -158,11 +158,12 @@ pub(crate) async fn run_statement_loop( continue; } Ok(TxnTaskOutcome::Staged(outcome)) => { - fold.fold_staged(exec, &plan, outcome)?; + fold.fold_staged(exec, &plan, outcome).await?; continue; } Ok(TxnTaskOutcome::CloneHandled(clone_response)) => { - fold.fold_answer(exec, &plan, clone_response.payload.as_ref())?; + fold.fold_answer(exec, &plan, clone_response.payload.as_ref()) + .await?; continue; } Ok(TxnTaskOutcome::InsteadOf) => { @@ -260,7 +261,9 @@ pub(crate) async fn run_statement_loop( } fold.last_lsn = task_response.watermark_lsn.as_u64(); - let task_rows = fold.fold_answer(exec, &plan, task_response.payload.as_ref())?; + let task_rows = fold + .fold_answer(exec, &plan, task_response.payload.as_ref()) + .await?; // Metered once per successfully dispatched task. if let Some(info) = &plan_metering_info { diff --git a/nodedb/src/control/server/shared/write_admission/admission_keys.rs b/nodedb/src/control/server/shared/write_admission/admission_keys.rs index 896c34f78..6b83a7c30 100644 --- a/nodedb/src/control/server/shared/write_admission/admission_keys.rs +++ b/nodedb/src/control/server/shared/write_admission/admission_keys.rs @@ -43,8 +43,8 @@ use crate::control::planner::calvin::submit::unique_claims::catalog_unique_claim use crate::control::planner::calvin::tx_class::write_keys::{WriteKeys, add_plan_write_keys}; use crate::control::planner::calvin::write_class::is_write_plan; use crate::control::state::SharedState; -use crate::control::wal_replication::transaction_redo::collections::written_collections; use crate::types::{DatabaseId, TenantId}; +use nodedb_physical::physical_plan::written_collections; /// The moded lock request of one write. #[derive(Debug, Default)] diff --git a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs index c3bfadaf3..45a3f7f89 100644 --- a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs +++ b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs @@ -672,6 +672,7 @@ mod tests { projection: Vec::new(), limit: 0, offset: 0, + system_as_of_ms: None, }), PhysicalPlan::Document(DocumentOp::DropIndex { collection: QualifiedCollection::new(DatabaseId::DEFAULT, "c"), diff --git a/nodedb/src/control/source_read/query.rs b/nodedb/src/control/source_read/query.rs index 8ac2eb492..9e9c1fadb 100644 --- a/nodedb/src/control/source_read/query.rs +++ b/nodedb/src/control/source_read/query.rs @@ -41,7 +41,8 @@ pub(crate) async fn read_query_rows( tenant_id: ctx.tenant_id, redaction: None, sequences: None, - })?; + }) + .await?; let ShapeOutcome::Rows(shaped) = shaped else { return Err(crate::Error::Internal { detail: format!( diff --git a/nodedb/src/control/wal_replication/transaction_redo/mod.rs b/nodedb/src/control/wal_replication/transaction_redo/mod.rs index 62a4091fd..b04186207 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/mod.rs +++ b/nodedb/src/control/wal_replication/transaction_redo/mod.rs @@ -7,13 +7,12 @@ //! - [`chunks`]: the streams a redo past the entry limit travels in. //! - [`propose`]: the proposal of a committed redo, inline or chunked. //! - [`abandon`]: the retry of a failed stream abandon. -//! - [`collections`] / [`sum_targets`]: what the payload derives from the -//! commit's buffered plans. +//! - [`sum_targets`]: what the payload derives from the commit's buffered +//! plans. pub mod abandon; pub mod apply; pub mod chunks; -pub mod collections; pub mod payload; pub mod propose; pub mod sum_targets; diff --git a/nodedb/src/control/wal_replication/transaction_redo/payload.rs b/nodedb/src/control/wal_replication/transaction_redo/payload.rs index 0e0bc0e8e..5a2d6579a 100644 --- a/nodedb/src/control/wal_replication/transaction_redo/payload.rs +++ b/nodedb/src/control/wal_replication/transaction_redo/payload.rs @@ -14,8 +14,8 @@ use crate::event::EventSource; use crate::types::{DatabaseId, TenantId}; use crate::wal::RedoRecord; -use super::collections::written_collections; use super::sum_targets::redo_sum_targets; +use nodedb_physical::physical_plan::written_collections; /// Everything a replica needs to apply one committed transaction's redo. #[derive(Debug, Clone)] diff --git a/nodedb/src/data/executor/core_loop/filter_match.rs b/nodedb/src/data/executor/core_loop/filter_match.rs index e00f371d8..2e9435549 100644 --- a/nodedb/src/data/executor/core_loop/filter_match.rs +++ b/nodedb/src/data/executor/core_loop/filter_match.rs @@ -23,6 +23,46 @@ use crate::types::TenantId; use super::CoreLoop; +/// The first filter-evaluation error a scan's infallible predicate met. +/// +/// Storage scans take a `bool` predicate. A fallible matcher runs through +/// [`Self::check`], which holds its first error and reads it as no match. +/// The caller returns the held error once the scan ends, so a division by +/// zero fails the statement instead of excluding the row. +#[derive(Default)] +pub(in crate::data::executor) struct HeldEvalError(std::cell::RefCell>); + +impl HeldEvalError { + /// The verdict of `eval`. After an error is held, no row is evaluated + /// and every row reads as no match. + pub(in crate::data::executor) fn check( + &self, + eval: impl FnOnce() -> Result, + ) -> bool { + if self.0.borrow().is_some() { + return false; + } + match eval() { + Ok(matched) => matched, + Err(e) => { + self.0.replace(Some(e)); + false + } + } + } + + /// `value`, or the held error when a predicate met one. + pub(in crate::data::executor) fn finish>( + &self, + value: T, + ) -> Result { + match self.0.borrow_mut().take() { + Some(e) => Err(E::from(e)), + None => Ok(value), + } + } +} + /// Evaluate `filters` against a stored row `body`, given the row's ALREADY /// RESOLVED strict schema (`Some` for a strict collection, `None` for /// schemaless). Callers that check many rows against the same @@ -97,16 +137,8 @@ impl CoreLoop { /// Build a reusable `Fn(&StorageKey, &[u8]) -> Result` /// closure evaluating `filters` against a stored row's `(row_key, body)`, /// resolving `collection`'s strict schema ONCE up front (not per row) and - /// capturing it in the closure. Suitable for a hot per-row scan loop - /// directly, or as the fallible half of a Cell-wrapping call pattern — - /// [`CoreLoop::merge_overlay_into_scan`] actually expects an - /// *infallible* `&dyn Fn(&StorageKey, &[u8]) -> bool`, not this - /// function's own `Result`-returning output, so callers that feed it - /// into that merge wrap the closure this function returns in a second, - /// infallible one that stashes any `Err` into a local - /// `Cell>` and checks it once the merge call returns — - /// see `stage_bulk_delete.rs`/`stage_bulk_update.rs`'s `raw_matches` / - /// `matches` pair for the exact pattern. + /// capturing it in the closure. A scan that takes an infallible predicate + /// wraps it with [`HeldEvalError::check`]. pub(in crate::data::executor) fn strict_aware_matcher<'a>( &self, database_id: u64, diff --git a/nodedb/src/data/executor/core_loop/vector_index_rebuild.rs b/nodedb/src/data/executor/core_loop/vector_index_rebuild.rs index 5fb63b3c8..607f666f8 100644 --- a/nodedb/src/data/executor/core_loop/vector_index_rebuild.rs +++ b/nodedb/src/data/executor/core_loop/vector_index_rebuild.rs @@ -73,20 +73,16 @@ impl CoreLoop { // Collect first (the scan borrows `&self.sparse`); re-index after // the borrow ends so `&mut self` is free for the HNSW insert. let mut docs: Vec<(nodedb_types::Surrogate, Vec)> = Vec::new(); - let scan = self.sparse.scan_documents_for_each( - db, - tenant_id, - &collection, - usize::MAX, - |doc_id, value| { - let normalized = crate::data::executor::scan_normalize::sparse_body_to_msgpack( - value, - body_format.as_format_ref(), - ); - docs.push((doc_id.surrogate(), normalized.into_owned())); - Ok(()) - }, - ); + // Each row at its current version, so a bitemporal collection's + // vectors are rebuilt too. + let scan = self.for_each_current_row(db, tenant_id, &collection, |doc_id, value| { + let normalized = crate::data::executor::scan_normalize::sparse_body_to_msgpack( + value, + body_format.as_format_ref(), + ); + docs.push((doc_id.surrogate(), normalized.into_owned())); + Ok(std::ops::ControlFlow::Continue(())) + }); if let Err(e) = scan { tracing::warn!( core = self.core_id, diff --git a/nodedb/src/data/executor/dispatch/document.rs b/nodedb/src/data/executor/dispatch/document.rs index 2e7fd636b..cc8477489 100644 --- a/nodedb/src/data/executor/dispatch/document.rs +++ b/nodedb/src/data/executor/dispatch/document.rs @@ -376,6 +376,7 @@ impl CoreLoop { projection, limit, offset, + system_as_of_ms, } => self.execute_document_indexed_fetch( task, super::super::handlers::document::index_fetch::IndexedFetchParams { @@ -387,6 +388,7 @@ impl CoreLoop { projection, limit: *limit, offset: *offset, + system_as_of_ms: *system_as_of_ms, }, ), diff --git a/nodedb/src/data/executor/enforcement/chain_guard.rs b/nodedb/src/data/executor/enforcement/chain_guard.rs index 542cfc027..6affcb4b8 100644 --- a/nodedb/src/data/executor/enforcement/chain_guard.rs +++ b/nodedb/src/data/executor/enforcement/chain_guard.rs @@ -409,12 +409,7 @@ impl CoreLoop { collection: &str, storage_key: &crate::engine::document::store::StorageKey, ) -> crate::Result>> { - if self.is_bitemporal(database_id, tid, collection) { - self.sparse - .versioned_get_current(database_id, tid, collection, storage_key) - } else { - self.sparse.get(database_id, tid, collection, storage_key) - } + self.current_row_body(database_id, tid, collection, storage_key) } /// The config of a hash-chained collection. diff --git a/nodedb/src/data/executor/enforcement/materialized_sum/divergence.rs b/nodedb/src/data/executor/enforcement/materialized_sum/divergence.rs index 35bbf3fa6..35ab008a7 100644 --- a/nodedb/src/data/executor/enforcement/materialized_sum/divergence.rs +++ b/nodedb/src/data/executor/enforcement/materialized_sum/divergence.rs @@ -155,8 +155,7 @@ impl CoreLoop { let mut rows: Vec = Vec::with_capacity(doc_ids.len()); for key in doc_ids { let Ok(Some(bytes)) = - self.sparse - .get(check.database_id, check.tid, check.collection, key) + self.current_row_body(check.database_id, check.tid, check.collection, key) else { continue; }; diff --git a/nodedb/src/data/executor/enforcement/materialized_sum/rmw.rs b/nodedb/src/data/executor/enforcement/materialized_sum/rmw.rs index f5c152afe..c576f08d8 100644 --- a/nodedb/src/data/executor/enforcement/materialized_sum/rmw.rs +++ b/nodedb/src/data/executor/enforcement/materialized_sum/rmw.rs @@ -214,31 +214,21 @@ impl CoreLoop { /// Read the target row's current stored bytes. /// - /// The plain read goes through the CALLER'S write transaction so a second - /// delta against the same row in the same transaction sees the first one's - /// result. A bitemporal target reads its current version the same way - /// `apply_point_put` reads its own pre-image. + /// The read goes through the CALLER'S write transaction so a second delta + /// against the same row in the same transaction sees the first one's + /// result, at the current version on a bitemporal target. fn read_balance_row( &self, txn: &WriteTransaction, params: &BalanceRmw<'_>, storage_key: &nodedb_types::StorageKey, ) -> crate::Result>> { - if self.is_bitemporal(params.database_id, params.tid, params.target_collection) { - self.sparse.versioned_get_current( - params.database_id, - params.tid, - params.target_collection, - storage_key, - ) - } else { - self.sparse.get_in_txn( - txn, - params.database_id, - params.tid, - params.target_collection, - storage_key, - ) - } + self.current_row_body_in_txn( + txn, + params.database_id, + params.tid, + params.target_collection, + storage_key, + ) } } diff --git a/nodedb/src/data/executor/enforcement/period_lock.rs b/nodedb/src/data/executor/enforcement/period_lock.rs index e3fac160f..8d29d26fc 100644 --- a/nodedb/src/data/executor/enforcement/period_lock.rs +++ b/nodedb/src/data/executor/enforcement/period_lock.rs @@ -9,10 +9,37 @@ use sonic_rs; use crate::bridge::envelope::ErrorCode; -use crate::engine::sparse::btree::SparseEngine; +use crate::data::executor::core_loop::CoreLoop; use nodedb_physical::physical_plan::{PeriodLockConfig, ResolvedSumTarget, resolved_sum_surrogate}; use nodedb_types::StorageKey; +/// Reads the current stored body of one row of a named reference collection. +pub type RefRowReader<'a> = dyn Fn(&str, &StorageKey) -> crate::Result>> + 'a; + +impl CoreLoop { + /// [`check_period_lock`] with the reference row read at its current + /// version, so a bitemporal reference collection resolves too. + pub(in crate::data::executor) fn check_period_lock_at( + &self, + database_id: u64, + tid: u64, + collection: &str, + doc_bytes: &[u8], + config: &PeriodLockConfig, + resolved_targets: &[ResolvedSumTarget], + ) -> Result<(), ErrorCode> { + check_period_lock( + &|ref_table: &str, key: &StorageKey| { + self.current_row_body(database_id, tid, ref_table, key) + }, + collection, + doc_bytes, + config, + resolved_targets, + ) + } +} + /// Check whether a write is allowed given the period lock configuration. /// /// `doc_bytes` is the document being written (INSERT/UPDATE) or the existing @@ -26,12 +53,12 @@ use nodedb_types::StorageKey; /// `(config.ref_table, period value)` — the same slot and lookup every other /// cross-collection resolution in this codebase uses. /// +/// `read_ref_row` reads the reference row by collection name and key. +/// /// Returns `Ok(())` if the write is allowed, or `Err(PeriodLocked)` if the /// period is closed, locked, or names no row the Control Plane could resolve. pub fn check_period_lock( - sparse: &SparseEngine, - database_id: u64, - tid: u64, + read_ref_row: &RefRowReader<'_>, collection: &str, doc_bytes: &[u8], config: &PeriodLockConfig, @@ -55,12 +82,7 @@ pub fn check_period_lock( collection: collection.to_string(), }); }; - let ref_bytes = match sparse.get( - database_id, - tid, - &config.ref_table, - &StorageKey::for_surrogate(surrogate), - ) { + let ref_bytes = match read_ref_row(&config.ref_table, &StorageKey::for_surrogate(surrogate)) { Ok(Some(bytes)) => bytes, Ok(None) => { // Period key not found in reference table — reject (unknown period). @@ -131,8 +153,16 @@ fn extract_field_string(bytes: &[u8], field_name: &str) -> Option { #[cfg(test)] mod tests { use super::*; + use crate::engine::sparse::btree::SparseEngine; use nodedb_types::Surrogate; + /// Reads a reference row from the plain table of `sparse`. + fn reader( + sparse: &SparseEngine, + ) -> impl Fn(&str, &StorageKey) -> crate::Result>> + '_ { + move |table: &str, key: &StorageKey| sparse.get(DB, TID, table, key) + } + fn make_config(allowed: &[&str]) -> PeriodLockConfig { PeriodLockConfig { period_column: "fiscal_period".into(), @@ -236,9 +266,7 @@ mod tests { assert!( check_period_lock( - &sparse, - DB, - TID, + &reader(&sparse), "journal_entries", &entry("2024-Q1"), &config, @@ -256,9 +284,7 @@ mod tests { let config = make_config(&["OPEN", "ADJUSTING"]); let err = check_period_lock( - &sparse, - DB, - TID, + &reader(&sparse), "journal_entries", &entry("2024-Q1"), &config, @@ -278,9 +304,7 @@ mod tests { let config = make_config(&["OPEN", "ADJUSTING"]); let err = check_period_lock( - &sparse, - DB, - TID, + &reader(&sparse), "journal_entries", &entry("2024-Q1"), &config, @@ -310,9 +334,7 @@ mod tests { let config = make_config(&["OPEN", "ADJUSTING"]); let err = check_period_lock( - &sparse, - DB, - TID, + &reader(&sparse), "journal_entries", &entry("2024-Q1"), &config, @@ -343,6 +365,6 @@ mod tests { let config = make_config(&["OPEN"]); let doc = nodedb_types::json_to_msgpack(&serde_json::json!({"amount": 100})).unwrap(); - assert!(check_period_lock(&sparse, DB, TID, "journal_entries", &doc, &config, &[]).is_ok()); + assert!(check_period_lock(&reader(&sparse), "journal_entries", &doc, &config, &[]).is_ok()); } } diff --git a/nodedb/src/data/executor/enforcement/statement.rs b/nodedb/src/data/executor/enforcement/statement.rs index 763fb5cd5..ed01e6409 100644 --- a/nodedb/src/data/executor/enforcement/statement.rs +++ b/nodedb/src/data/executor/enforcement/statement.rs @@ -88,7 +88,7 @@ impl CoreLoop { }; let mut entries = Vec::new(); for key in document_ids { - let Some(stored) = self.sparse.get(database_id, tid, collection, key)? else { + let Some(stored) = self.current_row_body(database_id, tid, collection, key)? else { continue; }; // A stored row of a collection that declares constraints over its diff --git a/nodedb/src/data/executor/handlers/bulk_dml/admission.rs b/nodedb/src/data/executor/handlers/bulk_dml/admission.rs index eca591263..5616e4f0d 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/admission.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/admission.rs @@ -71,8 +71,8 @@ impl CoreLoop { // or an edge appeared or disappeared among the matched docs — since then, // the wrong edge would be retracted and a stale one would dangle. The // surrogate-set check above cannot see this: the surrogate set is - // unchanged. This runs BEFORE any write, so `sparse.get` still returns - // pre-mutation content. + // unchanged. This runs BEFORE any write, so `current_row_body` still + // returns pre-mutation content. if let Some(predicted) = admission.predicted_edges { let actual = self.ollp_actual_edges(database_id, tid, admission.collection, &apply_ids); if !super::scan::ollp_edges_match(actual, predicted) { @@ -105,8 +105,8 @@ impl CoreLoop { /// Compute the sorted ACTUAL implicit-edge set for the matched docs. /// - /// For each matched storage key, fetch the stored doc bytes via the SAME - /// `sparse.get` path the delete loop uses, decode it, and — only when + /// For each matched storage key, read the current stored doc bytes via the + /// SAME `current_row_body` read the delete loop uses, decode it, and — only when /// it carries BOTH `_from` and `_to` as strings — record an /// [`OllpPredictedEdge`] with the raw `_type` as `label`. A matched doc /// without both endpoints is not an edge and is skipped; if it gained an @@ -125,32 +125,38 @@ impl CoreLoop { collection: &str, matching_ids: &[StorageKey], ) -> Vec { - // `decode_document` returns `serde_json::Value`, whose `get`/`as_str` - // are inherent methods — no extra trait import needed. let mut edges: Vec = Vec::new(); for key in matching_ids { - let Ok(Some(bytes)) = self.sparse.get(database_id, tid, collection, key) else { + let Ok(Some(bytes)) = self.current_row_body(database_id, tid, collection, key) else { continue; }; - let Ok(doc) = doc_format::decode_document(&bytes) else { - continue; - }; - let from = doc.get("_from").and_then(|v| v.as_str()); - let to = doc.get("_to").and_then(|v| v.as_str()); - if let (Some(from), Some(to)) = (from, to) { - let label = doc - .get("_type") - .and_then(|v| v.as_str()) - .map(str::to_string); - edges.push(OllpPredictedEdge { - surrogate: key.surrogate().as_u32(), - from: from.to_string(), - to: to.to_string(), - label, - }); - } + edges.extend(implicit_edge_of(*key, &bytes)); } edges.sort_unstable(); edges } } + +/// The implicit edge a stored edge document carries: both `_from` and `_to` +/// as strings, with the raw `_type` as label. `None` for any other row. +/// +/// `decode_document` returns `serde_json::Value`, whose `get`/`as_str` are +/// inherent methods. Bytes that do not decode, such as a strict Binary +/// Tuple, carry no edge. +pub(in crate::data::executor) fn implicit_edge_of( + key: StorageKey, + bytes: &[u8], +) -> Option { + let doc = doc_format::decode_document(bytes).ok()?; + let from = doc.get("_from").and_then(|v| v.as_str())?; + let to = doc.get("_to").and_then(|v| v.as_str())?; + Some(OllpPredictedEdge { + surrogate: key.surrogate().as_u32(), + from: from.to_string(), + to: to.to_string(), + label: doc + .get("_type") + .and_then(|v| v.as_str()) + .map(str::to_string), + }) +} diff --git a/nodedb/src/data/executor/handlers/bulk_dml/delete.rs b/nodedb/src/data/executor/handlers/bulk_dml/delete.rs index fccc93f5e..90df5a78f 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/delete.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/delete.rs @@ -3,7 +3,6 @@ use tracing::debug; use crate::bridge::envelope::{ErrorCode, Response, WriteSetEntry}; -use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::enforcement::write_hook; use crate::data::executor::handlers::partial_refusal::refusal_after_rows; @@ -17,6 +16,7 @@ use nodedb_physical::physical_plan::{ }; use super::delete_cascade::{BulkDeleteRowCascade, TextFlush}; +use crate::data::executor::handlers::point::delete_version::CurrentRowRemoval; /// OLLP prediction inputs threaded to `execute_bulk_delete`: the predicted /// matched-doc surrogate set and the predicted implicit-edge set. Both are @@ -79,32 +79,12 @@ impl CoreLoop { let database_id = task.request.database_id.as_u64(); // Empty `filter_bytes` means "no WHERE clause" — match every row. - let filters: Vec = if filter_bytes.is_empty() { - Vec::new() - } else { - match zerompk::from_msgpack(filter_bytes) { - Ok(f) => f, - Err(e) => { - return self.response_error( - task, - ErrorCode::Internal { - detail: format!("deserialize filters: {e}"), - }, - ); - } - } - }; - - let matching_ids = match self.scan_matching_documents( - task.request.database_id.as_u64(), - tid, - collection, - &filters, - ) { - Ok(ids) => ids, - Err(e) => { - return self.response_error(task, ErrorCode::from(e)); - } + // The matched set is each row's current version. + let matching_ids = match super::decode_bulk_filters(filter_bytes) + .and_then(|filters| self.current_matching_rows(database_id, tid, collection, &filters)) + { + Ok(rows) => rows.into_iter().map(|(key, _)| key).collect(), + Err(e) => return self.response_error(task, ErrorCode::from(e)), }; // Settle the apply set and run every leader-only pre-write @@ -185,7 +165,7 @@ impl CoreLoop { .and_then(|config| config.enforcement.period_lock.as_ref()); if gate_policy || period_lock.is_some() { for key in &apply_ids { - let stored = match self.sparse.get(database_id, tid, collection, key) { + let stored = match self.current_row_body(database_id, tid, collection, key) { Ok(Some(bytes)) => bytes, Ok(None) => continue, Err(e) => return self.response_error(task, e), @@ -204,16 +184,14 @@ impl CoreLoop { return self.response_error(task, e); } if let Some(lock) = period_lock - && let Err(e) = - crate::data::executor::enforcement::period_lock::check_period_lock( - &self.sparse, - database_id, - tid, - collection, - &stored, - lock, - resolved_sum_targets, - ) + && let Err(e) = self.check_period_lock_at( + database_id, + tid, + collection, + &stored, + lock, + resolved_sum_targets, + ) { return self.response_error(task, e); } @@ -251,6 +229,11 @@ impl CoreLoop { // Removed rows whose text has not left the inverted index yet. It // leaves in batches, one write transaction each. let mut pending_text: Vec = Vec::new(); + // One system time closes every matched row of a bitemporal + // collection, so the statement removes them at one instant. + let bitemporal_sys_from_ms = self + .is_bitemporal(database_id, tid, collection) + .then(|| self.bitemporal_now_ms()); for storage_key in &apply_ids { // Capture pre-deletion snapshot if RETURNING was requested, or if // the collection is indexed (needed to recompute the removed @@ -260,42 +243,37 @@ impl CoreLoop { // will not decode is a different answer: it would silently drop out // of RETURNING and, worse, contribute no removed index tuples, so // its old secondary-index entries would survive the delete. + // A read error is a refusal too: read as an absent row, it drops + // the row from RETURNING and keeps its index values unrecorded. let pre_delete_doc: Option = if returning.is_some() || !index_paths.is_empty() { - match self - .sparse - .get( - task.request.database_id.as_u64(), - tid, - collection, - storage_key, - ) - .ok() - .flatten() - { - Some(bytes) => { - let identity = storage_key.to_identity(); - match returning_doc::from_stored_json( - &bytes, - &identity, - strict_schema.as_ref(), - &identity_column, - ) { - Ok(doc) => Some(doc), - Err(e) => { - let code = refusal_after_rows(affected, e); - return self.bulk_delete_refusal( - task, - tid, - collection, - code, - &mut pending_text, - write_set, - ); - } - } + let decoded = self + .current_row_body(database_id, tid, collection, storage_key) + .and_then(|stored| { + stored + .map(|bytes| { + returning_doc::from_stored_json( + &bytes, + &storage_key.to_identity(), + strict_schema.as_ref(), + &identity_column, + ) + }) + .transpose() + }); + match decoded { + Ok(doc) => doc, + Err(e) => { + let code = refusal_after_rows(affected, e); + return self.bulk_delete_refusal( + task, + tid, + collection, + code, + &mut pending_text, + write_set, + ); } - None => None, } } else { None @@ -321,17 +299,33 @@ impl CoreLoop { ); } }; - let deleted_bytes = self - .sparse - .delete_in_txn( - &row_txn, - task.request.database_id.as_u64(), + // The row's current version leaves through the same per-row write + // a point delete lands. A delete error refuses the statement: + // read as an absent row, it leaves the row stored under a success. + let removed = match self.remove_current_row_in_txn( + &row_txn, + CurrentRowRemoval { + database_id, tid, collection, storage_key, - ) - .ok() - .flatten(); + bitemporal_sys_from_ms, + }, + ) { + Ok(removed) => removed, + Err(e) => { + let code = refusal_after_rows(affected, e); + return self.bulk_delete_refusal( + task, + tid, + collection, + code, + &mut pending_text, + write_set, + ); + } + }; + let deleted_bytes = removed.prior; let mut target_writes = Vec::new(); if let Some(bytes) = deleted_bytes.as_deref() { match write_hook::run( @@ -398,6 +392,7 @@ impl CoreLoop { collection, storage_key: *storage_key, deleted_bytes: bytes, + tombstone_sys_from_ms: removed.tombstone_sys_from_ms, strict_schema: strict_schema.as_ref(), declared_primary_key, has_vectors, diff --git a/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs b/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs index fd94e30ff..2605466b5 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/delete_cascade.rs @@ -22,6 +22,7 @@ use nodedb_types::columnar::StrictSchema; use crate::bridge::envelope::{ErrorCode, Response, WriteSetEntry}; use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::core_loop::redo_image::removed_row_image; use crate::data::executor::handlers::partial_refusal::refusal_after_rows; use crate::data::executor::handlers::point::apply_put::SpatialEntryId; use crate::data::executor::handlers::transaction::stage_write::stored_row_identity; @@ -84,6 +85,9 @@ pub(in crate::data::executor) struct BulkDeleteRowCascade<'a> { /// The row's pre-deletion bytes, as `sparse.delete` returned them — /// never re-read. pub deleted_bytes: &'a [u8], + /// The system time of the tombstone that closed a bitemporal row, `None` + /// for a row removed from the plain table. + pub tombstone_sys_from_ms: Option, /// The collection's strict schema, when it stores Binary Tuples. Decodes /// `deleted_bytes` so the row's identity column is readable. pub strict_schema: Option<&'a StrictSchema>, @@ -119,6 +123,7 @@ impl CoreLoop { collection, storage_key, deleted_bytes, + tombstone_sys_from_ms, strict_schema, declared_primary_key, has_vectors, @@ -199,10 +204,13 @@ impl CoreLoop { ); } // The removal, journalled after apply: the plan carries no - // pre-dispatch record of it. A delete carries no post-image body. - write_set.push(WriteSetEntry::delete( + // pre-dispatch record of it. A delete carries no post-image body. A + // bitemporal tombstone carries its version key, so replay closes the + // same version. + write_set.push(removed_row_image( row_surrogate.as_u32(), row_identity.clone(), + tombstone_sys_from_ms, )); // Emit a delete event per affected row to the Event Plane, so // AFTER-DELETE triggers and CDC/change-stream consumers see diff --git a/nodedb/src/data/executor/handlers/bulk_dml/mod.rs b/nodedb/src/data/executor/handlers/bulk_dml/mod.rs index 772811af6..9838966c3 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/mod.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/mod.rs @@ -9,10 +9,12 @@ pub mod admission; pub mod delete; pub mod delete_cascade; pub mod scan; +pub mod txn_match; pub mod update; pub mod update_persist; pub mod update_project; pub(in crate::data::executor) use admission::BulkAdmission; pub(in crate::data::executor) use delete::{BulkDeleteParams, OllpPrediction}; +pub(in crate::data::executor) use txn_match::{TxnPrediction, decode_bulk_filters}; pub(in crate::data::executor) use update::BulkUpdateParams; diff --git a/nodedb/src/data/executor/handlers/bulk_dml/scan.rs b/nodedb/src/data/executor/handlers/bulk_dml/scan.rs index ad8b34278..1d9e21eaf 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/scan.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/scan.rs @@ -1,65 +1,215 @@ // SPDX-License-Identifier: BUSL-1.1 +//! The rows a predicate write matches, and the OLLP helpers that compare +//! them with a carried prediction. +//! +//! The reads here are the one place a document read decides where a row's +//! current version lives. A bitemporal collection holds its rows only in the +//! versioned table, so the reads take each row's current version there. Any +//! other collection reads the plain document table. + +use std::ops::ControlFlow; + use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::core_loop::filter_match::HeldEvalError; +use crate::engine::sparse::btree_versioned::VersionedScanParams; +use crate::engine::sparse::scan_stop::never_stop; use nodedb_types::StorageKey; -use redb::{ReadableDatabase, ReadableTable}; + +/// One matched row: its storage key and its current stored body. +pub(in crate::data::executor) type MatchedRow = (StorageKey, Vec); + +/// Rows read per page of a bitemporal collection's current versions. +const BITEMPORAL_PAGE_ROWS: usize = 1024; impl CoreLoop { - /// Scan documents in a collection matching the given filters. + /// Each `(row key, current body)` in `collection` that `filters` match. /// - /// Returns the storage keys of all matching documents. - pub(in crate::data::executor) fn scan_matching_documents( + /// Autocommit bulk delete, update and TRUNCATE, facet counts and the + /// Calvin resolve read their matched set here. A transaction reads it through + /// [`Self::current_rows_where`] with its overlay folded in. A filter + /// evaluation error fails the read. + pub(in crate::data::executor) fn current_matching_rows( &self, database_id: u64, tid: u64, collection: &str, filters: &[ScanFilter], + ) -> crate::Result> { + let raw_matches = self.strict_aware_matcher(database_id, tid, collection, filters); + let held = HeldEvalError::default(); + let matches = |row_key: &StorageKey, body: &[u8]| held.check(|| raw_matches(row_key, body)); + let rows = self.current_rows_where(database_id, tid, collection, &matches)?; + held.finish(rows) + } + + /// The storage key of each row of `collection` at its current version. + pub(in crate::data::executor) fn current_row_keys( + &self, + database_id: u64, + tid: u64, + collection: &str, ) -> crate::Result> { - let prefix = crate::engine::sparse::btree::coll_prefix(database_id, tid, collection); - let end = format!("{prefix}\u{ffff}"); + let mut keys = Vec::new(); + self.for_each_current_row(database_id, tid, collection, |key, _| { + keys.push(*key); + Ok(ControlFlow::Continue(())) + })?; + Ok(keys) + } - let read_txn = self - .sparse - .db() - .begin_read() - .map_err(|e| crate::Error::Storage { - engine: "sparse".into(), - detail: format!("read txn: {e}"), - })?; - let table = read_txn - .open_table(crate::engine::sparse::btree::DOCUMENTS) - .map_err(|e| crate::Error::Storage { - engine: "sparse".into(), - detail: format!("open table: {e}"), - })?; - - // Resolve the strict schema ONCE for the whole scan rather than - // per row: `strict_aware_matcher` captures it in the closure so the - // `doc_configs` lookup doesn't repeat for every row in `range`. - let matches = self.strict_aware_matcher(database_id, tid, collection, filters); - - let mut ids = Vec::new(); - if let Ok(range) = table.range(prefix.as_str()..end.as_str()) { - for entry in range.flatten() { - let full_key = entry.0.value(); - let value_bytes = entry.1.value(); - let Some(rest) = full_key.strip_prefix(&prefix) else { - continue; - }; - let key = StorageKey::parse(rest).ok_or_else(|| { - crate::engine::sparse::btree::invalid_storage_key_err( - crate::engine::sparse::btree::KeyedTable::Documents, - collection, - rest, - ) - })?; - if matches(&key, value_bytes)? { - ids.push(key); + /// Each `(row key, current body)` in `collection` that `matches` accepts. + /// + /// A bitemporal collection yields the newest system-time version of each + /// row from the versioned table. A row whose newest version is a + /// tombstone is absent. Any other collection reads the plain table. + pub(in crate::data::executor) fn current_rows_where( + &self, + database_id: u64, + tid: u64, + collection: &str, + matches: &dyn Fn(&StorageKey, &[u8]) -> bool, + ) -> crate::Result> { + if self.is_bitemporal(database_id, tid, collection) { + return self.sparse.versioned_scan_as_of( + VersionedScanParams { + database_id, + tenant: tid, + coll: collection, + sys_cutoff_ms: None, + valid_at_ms: None, + limit: usize::MAX, + }, + matches, + &never_stop, + ); + } + let mut rows = Vec::new(); + self.sparse.scan_documents_for_each( + database_id, + tid, + collection, + usize::MAX, + |key, body| { + if matches(key, body) { + rows.push((*key, body.to_vec())); + } + Ok(()) + }, + )?; + Ok(rows) + } + + /// The first `limit` rows of `collection` in key order, each at its + /// current version, with no filter. + pub(in crate::data::executor) fn current_rows( + &self, + database_id: u64, + tid: u64, + collection: &str, + limit: usize, + ) -> crate::Result> { + if self.is_bitemporal(database_id, tid, collection) { + return self.sparse.versioned_scan_as_of( + VersionedScanParams { + database_id, + tenant: tid, + coll: collection, + sys_cutoff_ms: None, + valid_at_ms: None, + limit, + }, + &|_: &StorageKey, _: &[u8]| true, + &never_stop, + ); + } + self.sparse + .scan_documents(database_id, tid, collection, limit) + } + + /// Visit each row of `collection` at its current version, in key order, + /// until `visit` breaks. + /// + /// A plain collection streams one row at a time. A bitemporal collection + /// is read a page at a time, so at most one page of rows is in memory. A + /// visit error ends the scan and is returned. + pub(in crate::data::executor) fn for_each_current_row( + &self, + database_id: u64, + tid: u64, + collection: &str, + mut visit: impl FnMut(&StorageKey, &[u8]) -> crate::Result>, + ) -> crate::Result<()> { + if !self.is_bitemporal(database_id, tid, collection) { + return self + .sparse + .scan_documents_while(database_id, tid, collection, visit); + } + let mut after: Option = None; + loop { + let page = self.sparse.versioned_scan_as_of_after( + VersionedScanParams { + database_id, + tenant: tid, + coll: collection, + sys_cutoff_ms: None, + valid_at_ms: None, + limit: BITEMPORAL_PAGE_ROWS, + }, + after.as_ref(), + &|_: &StorageKey, _: &[u8]| true, + &never_stop, + )?; + let full = page.len() >= BITEMPORAL_PAGE_ROWS; + for (key, body) in &page { + if visit(key, body)?.is_break() { + return Ok(()); } } + match page.into_iter().next_back() { + Some((key, _)) if full => after = Some(key), + _ => return Ok(()), + } + } + } + + /// The current stored body of one row, `None` when the row is absent. + /// + /// This is the per-row form of the view [`Self::current_rows_where`] + /// scans. A predicate write reads each matched row's pre-image here. + pub(in crate::data::executor) fn current_row_body( + &self, + database_id: u64, + tid: u64, + collection: &str, + key: &StorageKey, + ) -> crate::Result>> { + if self.is_bitemporal(database_id, tid, collection) { + self.sparse + .versioned_get_current(database_id, tid, collection, key) + } else { + self.sparse.get(database_id, tid, collection, key) + } + } + + /// [`Self::current_row_body`] read inside `txn`, so it sees the writes + /// `txn` staged. + pub(in crate::data::executor) fn current_row_body_in_txn( + &self, + txn: &redb::WriteTransaction, + database_id: u64, + tid: u64, + collection: &str, + key: &StorageKey, + ) -> crate::Result>> { + if self.is_bitemporal(database_id, tid, collection) { + self.sparse + .versioned_get_current_in_txn(txn, database_id, tid, collection, key) + } else { + self.sparse + .get_in_txn(txn, database_id, tid, collection, key) } - Ok(ids) } } @@ -94,8 +244,8 @@ pub(in crate::data::executor) fn ollp_actual_surrogates(doc_ids: &[StorageKey]) /// True when the live `matching_ids` surrogate set equals the carried /// `predicted` set. Both sides are sorted before comparison, so the result is /// deterministic on every replica. Shared by the bulk-DML apply handlers and -/// the Calvin active-stage OLLP verifier so the `actual == predicted` guard -/// lives in exactly one place. +/// the transaction prediction check (`TxnPrediction::holds`) so the +/// `actual == predicted` guard lives in exactly one place. pub(in crate::data::executor) fn ollp_surrogates_match( matching_ids: &[StorageKey], predicted: &[u32], @@ -118,3 +268,302 @@ pub(in crate::data::executor) fn ollp_edges_match( predicted_sorted.sort_unstable(); actual == predicted_sorted } + +#[cfg(test)] +mod tests { + //! Predicate writes on a bitemporal collection: the shared read matches + //! each row's current version, and the bulk apply closes or supersedes + //! that version as the point paths do. + + use super::*; + use crate::bridge::envelope::{RowEffect, Status}; + use crate::bridge::scan_filter::FilterOp; + use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; + use crate::data::executor::doc_format; + use crate::data::executor::handlers::bulk_dml::{ + BulkDeleteParams, BulkUpdateParams, OllpPrediction, + }; + use crate::engine::document::store::{CollectionConfig, IndexPath}; + use crate::engine::sparse::btree_versioned::{VersionedIndexEntry, VersionedPut}; + use crate::types::{DatabaseId, TenantId}; + use nodedb_physical::physical_plan::UpdateValue; + use nodedb_types::Surrogate; + + const TID: u64 = 1; + const LEDGER: &str = "ledger"; + /// The system time every seeded version lands at, before any statement. + const SEEDED_AT: i64 = 1_000; + + fn key(surrogate: u32) -> StorageKey { + StorageKey::for_surrogate(Surrogate(surrogate)) + } + + /// A bitemporal `ledger` with a secondary index on `kind`, holding rows + /// 1 (`x`), 2 (`y`) and 3 (`x`) at `SEEDED_AT`. + fn bitemporal_ledger(core: &mut CoreLoop, database_id: u64) { + let mut config = CollectionConfig::new(LEDGER); + config.bitemporal = true; + config.index_paths.push(IndexPath::new("kind")); + core.doc_configs.insert( + ( + DatabaseId::new(database_id), + TenantId::new(TID), + LEDGER.to_string(), + ), + config, + ); + for (surrogate, kind) in [(1, "x"), (2, "y"), (3, "x")] { + let body = doc_format::encode_to_msgpack(&serde_json::json!({ "kind": kind })); + core.sparse + .versioned_put(VersionedPut { + database_id, + tenant: TID, + coll: LEDGER, + doc_id: &key(surrogate), + sys_from_ms: SEEDED_AT, + valid_from_ms: i64::MIN, + valid_until_ms: i64::MAX, + body: &body, + }) + .expect("seed version"); + core.sparse + .versioned_index_put(VersionedIndexEntry { + database_id, + tenant: TID, + coll: LEDGER, + field: "kind", + value: kind, + doc_id: &key(surrogate), + sys_from_ms: SEEDED_AT, + }) + .expect("seed index entry"); + } + } + + fn kind_eq(kind: &str) -> Vec { + vec![ScanFilter { + field: "kind".into(), + op: FilterOp::Eq, + value: nodedb_types::Value::String(kind.into()), + clauses: Vec::new(), + expr: None, + }] + } + + fn kind_eq_bytes(kind: &str) -> Vec { + zerompk::to_msgpack_vec(&kind_eq(kind)).expect("encode filter") + } + + /// The `kind` a stored body holds. + fn kind_of(body: &[u8]) -> String { + doc_format::decode_document(body) + .expect("row decodes") + .get("kind") + .and_then(|v| v.as_str()) + .expect("row carries a kind") + .to_string() + } + + fn version_at( + core: &CoreLoop, + database_id: u64, + surrogate: u32, + sys_cutoff_ms: Option, + ) -> Option> { + core.sparse + .versioned_get_as_of( + database_id, + TID, + LEDGER, + &key(surrogate), + sys_cutoff_ms, + None, + ) + .expect("read version") + } + + fn indexed_rows( + core: &CoreLoop, + database_id: u64, + kind: &str, + sys_cutoff_ms: Option, + ) -> Vec { + let mut rows: Vec = core + .sparse + .versioned_index_lookup_as_of(database_id, TID, LEDGER, "kind", kind, sys_cutoff_ms) + .expect("index lookup") + .into_iter() + .map(|k| k.surrogate().as_u32()) + .collect(); + rows.sort_unstable(); + rows + } + + fn affected(response: &crate::bridge::envelope::Response) -> u64 { + crate::control::server::shared::sql::staging_predicates::require_affected_count( + response.payload.as_bytes(), + ) + .expect("a bulk write reports an affected count") + } + + #[test] + fn the_shared_read_matches_current_versions_of_a_bitemporal_collection() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _req, _resp) = make_core_with_dir(dir.path()); + let database_id = make_default_task().request.database_id.as_u64(); + bitemporal_ledger(&mut core, database_id); + + let rows = core + .current_matching_rows(database_id, TID, LEDGER, &kind_eq("x")) + .expect("matching read"); + let surrogates: Vec = rows.iter().map(|(k, _)| k.surrogate().as_u32()).collect(); + assert_eq!(surrogates, vec![1, 3]); + assert!(rows.iter().all(|(_, body)| kind_of(body) == "x")); + assert_eq!( + core.current_row_body(database_id, TID, LEDGER, &key(2)) + .expect("row read") + .map(|body| kind_of(&body)), + Some("y".to_string()) + ); + } + + #[test] + fn bulk_delete_closes_the_current_version_of_each_matched_row() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _req, _resp) = make_core_with_dir(dir.path()); + let task = make_default_task(); + let database_id = task.request.database_id.as_u64(); + bitemporal_ledger(&mut core, database_id); + + let filter = kind_eq_bytes("x"); + let response = core.execute_bulk_delete( + &task, + TID, + BulkDeleteParams { + collection: LEDGER, + filter_bytes: &filter, + returning: None, + rls_filters: &[], + rls_write_check: &nodedb_types::RlsWriteCheck::NoPolicyApplies, + resolved_sum_targets: &[], + ollp: OllpPrediction { + surrogates: None, + edges: None, + }, + declared_primary_key: None, + }, + ); + + assert_eq!(response.status, Status::Ok, "{:?}", response.error_code); + assert_eq!(affected(&response), 2); + for surrogate in [1, 3] { + assert!( + version_at(&core, database_id, surrogate, None).is_none(), + "row {surrogate} has no current version" + ); + assert_eq!( + version_at(&core, database_id, surrogate, Some(SEEDED_AT)) + .map(|body| kind_of(&body)), + Some("x".to_string()), + "row {surrogate} keeps its history" + ); + } + assert_eq!( + version_at(&core, database_id, 2, None).map(|body| kind_of(&body)), + Some("y".to_string()) + ); + assert!(indexed_rows(&core, database_id, "x", None).is_empty()); + assert_eq!( + indexed_rows(&core, database_id, "x", Some(SEEDED_AT)), + vec![1, 3] + ); + + // Each removal journals its tombstone at its version key. + let tombstones: Vec<(u32, Option)> = response + .write_set + .iter() + .filter_map(|entry| match entry.effect { + RowEffect::Delete { system_from_ms } => Some((entry.surrogate, system_from_ms)), + _ => None, + }) + .collect(); + assert_eq!(tombstones.len(), 2); + assert!( + tombstones + .iter() + .all(|(_, sys)| sys.is_some_and(|sys| sys > SEEDED_AT)) + ); + } + + #[test] + fn bulk_update_writes_a_new_version_of_each_matched_row() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _req, _resp) = make_core_with_dir(dir.path()); + let task = make_default_task(); + let database_id = task.request.database_id.as_u64(); + bitemporal_ledger(&mut core, database_id); + + let filter = kind_eq_bytes("y"); + let updates = vec![( + "kind".to_string(), + UpdateValue::Literal( + nodedb_types::json_to_msgpack(&serde_json::json!("z")).expect("encode"), + ), + )]; + let response = core.execute_bulk_update( + &task, + TID, + BulkUpdateParams { + collection: LEDGER, + filter_bytes: &filter, + updates: &updates, + returning: None, + ollp_predicted_surrogates: None, + ollp_predicted_edges: None, + rls_filters: &[], + rls_write_check: &nodedb_types::RlsWriteCheck::NoPolicyApplies, + resolved_sum_targets: &[], + declared_primary_key: None, + }, + ); + + assert_eq!(response.status, Status::Ok, "{:?}", response.error_code); + assert_eq!(affected(&response), 1); + assert_eq!( + version_at(&core, database_id, 2, None).map(|body| kind_of(&body)), + Some("z".to_string()) + ); + assert_eq!( + version_at(&core, database_id, 2, Some(SEEDED_AT)).map(|body| kind_of(&body)), + Some("y".to_string()), + "the superseded version stays readable" + ); + for surrogate in [1, 3] { + assert_eq!( + version_at(&core, database_id, surrogate, None).map(|body| kind_of(&body)), + Some("x".to_string()) + ); + } + assert!(indexed_rows(&core, database_id, "y", None).is_empty()); + assert_eq!(indexed_rows(&core, database_id, "z", None), vec![2]); + assert_eq!( + indexed_rows(&core, database_id, "y", Some(SEEDED_AT)), + vec![2] + ); + + // The new version journals at its version key. + let versions: Vec = response + .write_set + .iter() + .filter_map(|entry| match &entry.effect { + RowEffect::Put { + version: Some(version), + .. + } => Some(version.sys_from_ms), + _ => None, + }) + .collect(); + assert_eq!(versions.len(), 1); + assert!(versions[0] > SEEDED_AT); + } +} diff --git a/nodedb/src/data/executor/handlers/bulk_dml/txn_match.rs b/nodedb/src/data/executor/handlers/bulk_dml/txn_match.rs new file mode 100644 index 000000000..f0615fd4e --- /dev/null +++ b/nodedb/src/data/executor/handlers/bulk_dml/txn_match.rs @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The rows a predicate write matches inside a transaction, and the check of +//! a carried prediction against them. +//! +//! A session stage and a Calvin stage both read BASE with the transaction's +//! overlay folded in. A Calvin overlay holds the plans staged before this +//! one, so the check runs at the plan's own turn in the transaction. BASE is +//! the current version of each row, read by [`CoreLoop::current_rows_where`]. + +use nodedb_physical::physical_plan::OllpPredictedEdge; +use nodedb_types::StorageKey; + +use super::admission::implicit_edge_of; +use super::scan::{ollp_edges_match, ollp_surrogates_match}; +use crate::bridge::scan_filter::ScanFilter; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::core_loop::filter_match::HeldEvalError; +use crate::types::{DatabaseId, TenantId, TxnId}; + +/// The rows a bulk plan predicts it writes: the matched surrogates, and the +/// implicit edges of the matched edge documents when the plan carries them. +#[derive(Clone, Copy)] +pub(in crate::data::executor) struct TxnPrediction<'a> { + pub surrogates: &'a [u32], + pub edges: Option<&'a [OllpPredictedEdge]>, +} + +impl<'a> TxnPrediction<'a> { + /// The prediction a plan carries. `None` when it carries no surrogate set. + pub(in crate::data::executor) fn of( + surrogates: &'a Option>, + edges: &'a Option>, + ) -> Option { + surrogates.as_deref().map(|surrogates| Self { + surrogates, + edges: edges.as_deref(), + }) + } + + /// Whether `rows` are exactly the predicted rows, carrying exactly the + /// predicted implicit edges. + pub(in crate::data::executor) fn holds(&self, rows: &[(StorageKey, Vec)]) -> bool { + let keys: Vec = rows.iter().map(|(key, _)| *key).collect(); + if !ollp_surrogates_match(&keys, self.surrogates) { + return false; + } + let Some(predicted) = self.edges else { + return true; + }; + let actual = rows + .iter() + .filter_map(|(key, body)| implicit_edge_of(*key, body)) + .collect(); + ollp_edges_match(actual, predicted) + } +} + +/// Decode a bulk plan's serialized filters. Empty bytes match every row. +pub(in crate::data::executor) fn decode_bulk_filters( + filter_bytes: &[u8], +) -> crate::Result> { + if filter_bytes.is_empty() { + return Ok(Vec::new()); + } + zerompk::from_msgpack(filter_bytes).map_err(|e| crate::Error::Serialization { + format: "msgpack".to_string(), + detail: format!("bulk predicate filters: {e}"), + }) +} + +impl CoreLoop { + /// Each `(row key, body)` that `filters` matches in the collection + /// `coll_key` names, as transaction `txn_id` sees it. + pub(in crate::data::executor) fn txn_matching_rows( + &self, + txn_id: TxnId, + coll_key: &(DatabaseId, TenantId, String), + filters: &[ScanFilter], + ) -> crate::Result)>> { + let (database_id, tenant, collection) = coll_key; + let (database_id, tid) = (database_id.as_u64(), tenant.as_u64()); + let collection = collection.as_str(); + let raw_matches = self.strict_aware_matcher(database_id, tid, collection, filters); + let held = HeldEvalError::default(); + let matches = |row_key: &StorageKey, body: &[u8]| held.check(|| raw_matches(row_key, body)); + let mut rows = self.current_rows_where(database_id, tid, collection, &matches)?; + self.merge_overlay_into_scan(txn_id, coll_key, &mut rows, &matches); + held.finish(rows) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_types::Surrogate; + + fn key(surrogate: u32) -> StorageKey { + StorageKey::for_surrogate(Surrogate::new(surrogate)) + } + + fn edge_body(from: &str, to: &str) -> Vec { + let mut obj = std::collections::HashMap::new(); + obj.insert( + "_from".to_string(), + nodedb_types::Value::String(from.to_string()), + ); + obj.insert( + "_to".to_string(), + nodedb_types::Value::String(to.to_string()), + ); + nodedb_types::value_to_msgpack(&nodedb_types::Value::Object(obj)).expect("encode") + } + + fn edge(surrogate: u32, from: &str, to: &str) -> OllpPredictedEdge { + OllpPredictedEdge { + surrogate, + from: from.to_string(), + to: to.to_string(), + label: None, + } + } + + #[test] + fn prediction_holds_on_exactly_the_predicted_rows() { + let rows = vec![(key(2), edge_body("a", "b")), (key(1), edge_body("c", "d"))]; + let surrogates = Some(vec![1, 2]); + let edges = Some(vec![edge(1, "c", "d"), edge(2, "a", "b")]); + let prediction = TxnPrediction::of(&surrogates, &edges).expect("prediction"); + assert!(prediction.holds(&rows)); + } + + #[test] + fn prediction_fails_on_an_extra_row() { + let rows = vec![(key(1), edge_body("c", "d")), (key(3), edge_body("e", "f"))]; + let surrogates = Some(vec![1]); + let prediction = TxnPrediction::of(&surrogates, &None).expect("prediction"); + assert!(!prediction.holds(&rows)); + } + + #[test] + fn prediction_fails_on_a_moved_implicit_edge() { + let rows = vec![(key(1), edge_body("c", "z"))]; + let surrogates = Some(vec![1]); + let edges = Some(vec![edge(1, "c", "d")]); + let prediction = TxnPrediction::of(&surrogates, &edges).expect("prediction"); + assert!(!prediction.holds(&rows)); + } + + #[test] + fn a_plan_without_surrogates_carries_no_prediction() { + assert!(TxnPrediction::of(&None, &None).is_none()); + } +} diff --git a/nodedb/src/data/executor/handlers/bulk_dml/update.rs b/nodedb/src/data/executor/handlers/bulk_dml/update.rs index 52cf20d79..c83ea12f8 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/update.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/update.rs @@ -3,14 +3,13 @@ use tracing::debug; use crate::bridge::envelope::{ErrorCode, Response, WriteSetEntry}; -use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::redo_image::StoredRow; use crate::data::executor::enforcement::unique::PostImage; use crate::data::executor::enforcement::write_hook; use crate::data::executor::handlers::partial_refusal::refusal_after_partial_apply; use crate::data::executor::handlers::point::update_reindex::NonbitemporalUpdateReindex; -use crate::data::executor::handlers::point::update_reindex_vector::UpdateVectorReindex; +use crate::data::executor::handlers::point::update_reindex_secondary::UpdateSecondaryReindex; use crate::data::executor::handlers::returning_doc; use crate::data::executor::handlers::returning_rows; use crate::data::executor::handlers::transaction::stage_write::stored_row_identity; @@ -95,39 +94,20 @@ impl CoreLoop { } // Empty `filter_bytes` means "no WHERE clause" — match every row. - let filters: Vec = if filter_bytes.is_empty() { - Vec::new() - } else { - match zerompk::from_msgpack(filter_bytes) { - Ok(f) => f, - Err(e) => { - return self.response_error( - task, - ErrorCode::Internal { - detail: format!("deserialize filters: {e}"), - }, - ); - } - } - }; - - let matching_ids = match self.scan_matching_documents( - task.request.database_id.as_u64(), - tid, - collection, - &filters, - ) { - Ok(ids) => ids, - Err(e) => { - return self.response_error(task, ErrorCode::from(e)); - } + // The matched set is each row's current version. + let database_id = task.request.database_id.as_u64(); + let matching_ids = match super::decode_bulk_filters(filter_bytes) + .and_then(|filters| self.current_matching_rows(database_id, tid, collection, &filters)) + { + Ok(rows) => rows.into_iter().map(|(key, _)| key).collect(), + Err(e) => return self.response_error(task, ErrorCode::from(e)), }; // Settle the apply set and run every leader-only pre-write // verification the plan's predictions call for. Any divergence returns // here, before the first row is touched. let apply_ids = match self.admit_bulk_predicate_write( - task.request.database_id.as_u64(), + database_id, tid, matching_ids, &super::BulkAdmission { @@ -159,14 +139,19 @@ impl CoreLoop { // HNSW vectors — the btree/FTS/graph reconciliation this handler already // does never touches the vector index, so KNN search would keep scoring // the stale pre-update embedding in the same process. - let database_id = task.request.database_id.as_u64(); let has_vectors = self.collection_has_vectors(database_id, tid, collection); - // The plain `INDEXES` secondary-index paths for this collection, cloned - // once for the whole statement. Each row's primary write reconciles - // these atomically via `nonbitemporal_update_reindex` so a value the - // UPDATE changed can't leave a stale index entry pointing at the old - // value (which would make a later lookup on the new value miss the row). + // One system time stamps every new version of a bitemporal + // collection, so the statement writes its rows at one instant. + let bitemporal_sys_from_ms = self + .is_bitemporal(database_id, tid, collection) + .then(|| self.bitemporal_now_ms()); + + // The secondary-index paths for this collection, cloned once for the + // whole statement. Each row's primary write reconciles them in the + // row's transaction, so a value the UPDATE changed can't leave a stale + // index entry pointing at the old value. A bitemporal collection + // reconciles the versioned index, any other the plain `INDEXES`. let index_paths = self .doc_configs .get(&config_key) @@ -198,6 +183,7 @@ impl CoreLoop { updates, strict_schema: strict_schema.as_ref(), declared_primary_key, + bitemporal_sys_from_ms, }) { Ok(projected) => projected, Err(e) => return self.response_error(task, e), @@ -277,6 +263,7 @@ impl CoreLoop { deferred_sum_targets: &[], wal_lsn: task.wal_lsn(), }, + bitemporal_sys_from_ms, ); let (touched, target_writes) = match persisted { Ok(persisted) => (persisted.touched, persisted.target_writes), @@ -300,7 +287,8 @@ impl CoreLoop { storage_key, ); // The row's post-image, then one durable redo entry per derived - // target row, naming the TARGET collection. + // target row, naming the TARGET collection. A bitemporal version + // carries its system time, so replay lands the same version. let image = self.stored_row_image( StoredRow { database_id, @@ -310,7 +298,7 @@ impl CoreLoop { identity: row_identity.clone(), }, &updated_bytes, - None, + bitemporal_sys_from_ms, ); match image { Ok(image) => write_set.push(image), @@ -344,21 +332,19 @@ impl CoreLoop { // surrogate + collection. let surrogate = storage_key.surrogate(); self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); - // Re-index the row's vectors from the new body (soft-delete the - // old HNSW node + insert the new one, keyed by the stable - // surrogate). No-op unless the collection has a vector field - // (gated above). - if has_vectors - && let Err(e) = self.update_reindex_vector_indexes(UpdateVectorReindex { - database_id, - tid, - collection, - storage_key, - new_body: &updated_bytes, - is_strict: strict_schema.is_some(), - has_vectors, - }) - { + // Re-index the row's dense vectors and sparse-vector postings + // from the new body, keyed by the stable surrogate, as the point + // update does. Each half is a no-op when the collection declares + // nothing of that kind. + if let Err(e) = self.update_reindex_vector_and_sparse(UpdateSecondaryReindex { + database_id, + tid, + collection, + storage_key, + new_body: &updated_bytes, + is_strict: strict_schema.is_some(), + has_vectors, + }) { // The row's body already committed. let code = refusal_after_partial_apply(ErrorCode::from(e)); return self.refusal_with_landed_rows(task, code, write_set); @@ -426,6 +412,7 @@ impl CoreLoop { mod tests { use super::*; use crate::bridge::envelope::Status; + use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; use crate::data::executor::doc_format; use nodedb_physical::physical_plan::UpdateValue; diff --git a/nodedb/src/data/executor/handlers/bulk_dml/update_persist.rs b/nodedb/src/data/executor/handlers/bulk_dml/update_persist.rs index 90ab249ba..a3b24d293 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/update_persist.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/update_persist.rs @@ -21,7 +21,9 @@ use crate::data::executor::enforcement::images::RowImages; use crate::data::executor::enforcement::materialized_sum::apply::TargetWrite; use crate::data::executor::enforcement::write_hook::{self, HookCtx}; use crate::data::executor::enforcement::{funnel, images}; -use crate::data::executor::handlers::point::update_reindex::NonbitemporalUpdateReindex; +use crate::data::executor::handlers::point::update_reindex::{ + BitemporalUpdateReindex, NonbitemporalUpdateReindex, +}; /// What landing one bulk-UPDATE row produced for the caller to account for. pub(super) struct PersistedBulkUpdateRow { @@ -44,10 +46,15 @@ impl CoreLoop { /// commit. The transaction is dropped un-committed, so the row leaves /// nothing behind. A row is never skipped, because a skipped row would /// report a smaller affected count than the predicate matched. + /// + /// `bitemporal_sys_from_ms` is `Some` on a bitemporal collection. The row + /// then lands as a new version valid for all time, with the versioned + /// index diff, through the same write the point update lands. pub(super) fn persist_bulk_update_row( &mut self, p: NonbitemporalUpdateReindex<'_>, hook: &HookCtx<'_>, + bitemporal_sys_from_ms: Option, ) -> crate::Result { // Hoisted before the transaction opens: the images below borrow them, // and a collection that declares no image-folding enforcement must not @@ -68,7 +75,25 @@ impl CoreLoop { surrogate: doc_id.surrogate(), new_doc, }; - let touched = self.nonbitemporal_update_reindex(&txn, p)?; + let touched = match bitemporal_sys_from_ms { + Some(sys_from_ms) => self.bitemporal_update_reindex( + &txn, + BitemporalUpdateReindex { + database_id: p.database_id, + tid: p.tid, + collection: p.collection, + doc_id: p.storage_key, + sys_from_ms, + valid_from_ms: i64::MIN, + valid_until_ms: i64::MAX, + new_body: p.new_body, + index_paths: p.index_paths, + old_doc: Some(old_doc), + new_doc, + }, + )?, + None => self.nonbitemporal_update_reindex(&txn, p)?, + }; // The row's postings follow its new text in the same transaction. self.update_reindex_text(&txn, text)?; diff --git a/nodedb/src/data/executor/handlers/bulk_dml/update_project.rs b/nodedb/src/data/executor/handlers/bulk_dml/update_project.rs index bceb4ef5e..47e1fe3ca 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/update_project.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/update_project.rs @@ -45,6 +45,10 @@ pub(in crate::data::executor) struct ProjectUpdateRows<'a> { /// Declared `PRIMARY KEY` column of a schemaless collection, `None` /// otherwise. `Some` makes the post-image guard below run. pub(in crate::data::executor) declared_primary_key: Option<&'a str>, + /// The system time of the new versions on a bitemporal collection, + /// `None` on any other. A bitemporal strict tuple carries it in its + /// reserved slot. + pub(in crate::data::executor) bitemporal_sys_from_ms: Option, } /// What every row of one update projects under. @@ -56,12 +60,14 @@ struct RowShape<'a> { updates: &'a [(String, UpdateValue)], strict_schema: Option<&'a StrictSchema>, declared_primary_key: Option<&'a str>, + bitemporal_sys_from_ms: Option, } /// The rows an update projects. pub(in crate::data::executor) enum MatchedRows<'a> { - /// Row keys whose current bodies are read from the store. A key whose - /// row is gone is skipped. + /// Row keys whose current bodies are read from the store, at the + /// current version on a bitemporal collection. A key whose row is gone + /// is skipped. Stored(&'a [StorageKey]), /// Rows with the current bodies the caller already read, such as a /// transaction's own staged writes merged over the store. @@ -84,6 +90,7 @@ impl CoreLoop { updates, strict_schema, declared_primary_key, + bitemporal_sys_from_ms, } = p; let shape = RowShape { database_id, @@ -92,13 +99,16 @@ impl CoreLoop { updates, strict_schema, declared_primary_key, + bitemporal_sys_from_ms, }; let rows = match rows { MatchedRows::Read(rows) => rows, MatchedRows::Stored(keys) => { let mut rows = Vec::with_capacity(keys.len()); for key in keys { - if let Some(current) = self.sparse.get(database_id, tid, collection, key)? { + if let Some(current) = + self.current_row_body(database_id, tid, collection, key)? + { rows.push((*key, current)); } } @@ -127,6 +137,7 @@ impl CoreLoop { updates, strict_schema, declared_primary_key, + bitemporal_sys_from_ms, } = shape; let config_key = ( DatabaseId::new(database_id), @@ -247,14 +258,27 @@ impl CoreLoop { // Re-encode — format depends on storage mode. An encode error // carries its own typed cause, such as a field the strict schema // does not declare. - let updated_bytes = match strict_schema { - Some(schema) => { + // A bitemporal strict tuple stamps the version's system time, valid + // for all time, as the point update encodes it. + let updated_bytes = match (strict_schema, bitemporal_sys_from_ms) { + (Some(schema), Some(sys_from_ms)) if schema.bitemporal => { + let ndb_val: nodedb_types::Value = doc.clone().into(); + crate::data::executor::strict_format::value_to_binary_tuple_bitemporal( + &ndb_val, + schema, + sys_from_ms, + i64::MIN, + i64::MAX, + collection, + )? + } + (Some(schema), _) => { let ndb_val: nodedb_types::Value = doc.clone().into(); crate::data::executor::strict_format::value_to_binary_tuple( &ndb_val, schema, collection, )? } - None => doc_format::encode_to_msgpack(&doc), + (None, _) => doc_format::encode_to_msgpack(&doc), }; Ok(ProjectedUpdateRow { @@ -312,8 +336,7 @@ impl CoreLoop { // that assigns the period column into it. if let Some(lock) = period_lock { for image in [&row.current_bytes, &row.updated_bytes] { - crate::data::executor::enforcement::period_lock::check_period_lock( - &self.sparse, + self.check_period_lock_at( database_id, tid, collection, diff --git a/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs b/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs index ea73dc536..26f579c3e 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/active_passive.rs @@ -83,12 +83,9 @@ impl CoreLoop { /// `commit_pending`, so a subsequent `CalvinResolve` builds one redo record /// and [`CoreLoop::install_calvin_redo`] installs it. /// - /// The one divergence from the static path: OLLP predicate verification - /// (leader-only) runs HERE, before staging, via - /// [`CoreLoop::verify_calvin_active_ollp`]. The dependent-read path has no - /// write-versioned read-set to vote on; its conflict detector is the OLLP - /// `actual != predicted` re-check. A mismatch returns `OllpRetryRequired` - /// and stages nothing, so no redo entry exists for it. The Control + /// A predicate plan checks its OLLP prediction as it stages, on this path + /// and the static one alike. A mismatch returns `OllpRetryRequired` and + /// drops everything staged, so no redo entry exists for it. The Control /// Plane scheduler releases locks and re-recons on `OllpRetryRequired`. /// /// The coordinator bakes the values it read into concrete point ops, so @@ -129,16 +126,6 @@ impl CoreLoop { ) .entered(); - // OLLP verification runs HERE, before staging, so a predicate-drift - // mismatch surfaces on THIS stage response (where the scheduler releases - // locks and re-recons) and nothing is staged or resolved. - let verified = self.verify_calvin_active_ollp(task, tenant_id.as_u64(), plans); - match verified { - Ok(true) => {} - Ok(false) => return self.calvin_stage_refusal(task, ErrorCode::OllpRetryRequired), - Err(e) => return self.calvin_stage_refusal(task, e.into()), - } - let synthetic_txn_id = match calvin_synthetic_txn_id(epoch, position, vshard_id) { Ok(id) => id, Err(e) => return self.calvin_stage_failure(task, epoch, position, vshard_id, e), @@ -399,9 +386,8 @@ mod tests { /// On the data-group leader, a predicate-write plan whose carried OLLP /// predicted set no longer matches live state returns `OllpRetryRequired` - /// BEFORE staging anything — so no stale redo is WAL-appended and the - /// coordinator can re-recon under a fresh attempt. Verifying at stage time - /// (not install) is the one divergence from the static path. + /// and keeps nothing staged — so no stale redo is WAL-appended and the + /// coordinator can re-recon under a fresh attempt. #[test] fn calvin_execute_active_ollp_drift_returns_retry_and_stages_nothing() { let dir = tempfile::tempdir().unwrap(); diff --git a/nodedb/src/data/executor/handlers/control/calvin/shared.rs b/nodedb/src/data/executor/handlers/control/calvin/shared.rs index 540f694e6..624afafff 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/shared.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/shared.rs @@ -22,9 +22,40 @@ impl CoreLoop { /// Refuse a Calvin stage that staged nothing, with the vote `error` names. /// /// `OllpRetryRequired` votes `PredictionDrift`. Every other error votes - /// `ParticipantError`. - pub(super) fn calvin_stage_refusal(&self, task: &ExecutionTask, error: ErrorCode) -> Response { + /// `ParticipantError`. Only the coordinator's client receives the error, + /// so the refusing node records it here. Every refusal is logged. An + /// internal error, which no client input causes, is also captured. + pub(super) fn calvin_stage_refusal( + &self, + task: &ExecutionTask, + epoch: u64, + position: u32, + error: ErrorCode, + ) -> Response { let vote = StageVote::of_stage_error(&error); + let vshard_id = task.request.vshard_id.as_u32(); + if vote == StageVote::PredictionDrift { + tracing::debug!( + core = self.core_id, + vshard_id, + epoch, + position, + ?error, + "calvin stage refused: a predicted state moved" + ); + } else { + tracing::warn!( + core = self.core_id, + vshard_id, + epoch, + position, + ?error, + "calvin stage refused: this slice aborts the transaction" + ); + } + if let ErrorCode::Internal { detail } = &error { + crate::diag::calvin_stage_refused_internal(vshard_id, epoch, position, detail); + } let mut response = self.response_error(task, error); response.stage_vote = Some(vote); response @@ -49,7 +80,7 @@ impl CoreLoop { self.drop_calvin_synthetic_overlay(epoch, position, vshard_id); // The scheduler proposes this abort vote and waits for the global // verdict before it drops anything. - self.calvin_stage_refusal(task, error.into()) + self.calvin_stage_refusal(task, epoch, position, error.into()) } } @@ -166,6 +197,29 @@ pub(in crate::data::executor) mod test_support { }) } + /// A match-all `BulkUpdate` setting `a` to `"2"`. + pub(in crate::data::executor) fn bulk_update_plan( + collection: &str, + predicted: Option>, + ) -> PhysicalPlan { + let literal = nodedb_types::json_to_msgpack(&serde_json::json!("2")).expect("literal"); + PhysicalPlan::Document(DocumentOp::BulkUpdate { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, collection), + filters: Vec::new(), + updates: vec![( + "a".to_string(), + nodedb_physical::physical_plan::UpdateValue::Literal(literal), + )], + returning: None, + ollp_predicted_surrogates: predicted, + ollp_predicted_edges: None, + rls_filters: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::NoPolicyApplies, + resolved_sum_targets: Vec::new(), + declared_primary_key: None, + }) + } + /// Seed a row directly into base storage (bypassing Calvin staging), the /// pre-existing state the active-path OLLP verifier scans against. pub(in crate::data::executor) fn seed_row( @@ -216,6 +270,8 @@ mod tests { let response = core.calvin_stage_refusal( &task, + 1, + 0, ErrorCode::Internal { detail: "stage".into(), }, diff --git a/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs b/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs index 6738bea4a..009a46cc9 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/static_stage.rs @@ -171,17 +171,7 @@ impl CoreLoop { } else { task.request.event_source }; - let database_id = task.request.database_id; - let collections = - crate::control::wal_replication::transaction_redo::collections::written_collections( - std::slice::from_ref(plan), - ); - self.txn_overlay_mut(txn_id).begin_staging( - source, - collections - .into_iter() - .map(|collection| (database_id, tenant_id, collection)), - ); + self.begin_plan_staging(txn_id, source, task.request.database_id, tenant_id, plan); } } @@ -197,9 +187,144 @@ mod tests { use crate::types::DatabaseId; use super::super::shared::test_support::{ - bulk_delete_plan, canonical_ilp_plan, doc_value, make_task, point_insert_plan, + bulk_delete_plan, bulk_update_plan, canonical_ilp_plan, doc_value, make_task, + point_insert_plan, seed_row, }; + fn stage_at(core: &mut CoreLoop, epoch: u64, plans: &[PhysicalPlan]) -> Response { + let ctx = CalvinExecCtx { + epoch, + position: 0, + epoch_system_ms: 0, + }; + core.execute_calvin_execute_static(&make_task(), ctx, &TenantId::new(1), plans, &[], &[]) + } + + fn assert_drift_stages_nothing(core: &CoreLoop, response: &Response, epoch: u64) { + assert_eq!(response.status, Status::Error); + assert_eq!( + response.error_code.as_deref(), + Some(&ErrorCode::OllpRetryRequired) + ); + assert_eq!(response.stage_vote, Some(StageVote::PredictionDrift)); + let vshard = make_task().request.vshard_id.as_u32(); + assert!(!core.calvin.commit_pending.contains_key(&(epoch, 0, vshard))); + let synthetic = calvin_synthetic_txn_id(epoch, 0, vshard).unwrap(); + assert!(!core.txn_overlays.contains_key(&synthetic)); + } + + /// A bulk delete whose predicted set misses a matching row votes + /// `PredictionDrift` on the static path and keeps nothing staged. + #[test] + fn static_stage_bulk_delete_drift_votes_prediction_drift_and_stages_nothing() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + seed_row(&mut core, "orders", 10); + seed_row(&mut core, "orders", 11); + + let response = stage_at(&mut core, 31, &[bulk_delete_plan("orders", Some(vec![10]))]); + + assert_drift_stages_nothing(&core, &response, 31); + } + + #[test] + fn static_stage_bulk_update_drift_votes_prediction_drift() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + seed_row(&mut core, "orders", 10); + seed_row(&mut core, "orders", 11); + + let response = stage_at(&mut core, 32, &[bulk_update_plan("orders", Some(vec![11]))]); + + assert_drift_stages_nothing(&core, &response, 32); + } + + /// The prediction is checked at the plan's turn: a row an earlier plan of + /// the same transaction staged is one the predicate matches. + #[test] + fn static_stage_bulk_delete_matches_rows_an_earlier_plan_staged() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + + let response = stage_at( + &mut core, + 33, + &[ + point_insert_plan("orders", "o1", 12), + bulk_delete_plan("orders", Some(vec![12])), + ], + ); + + assert_eq!(response.status, Status::Ok, "{:?}", response.error_code); + assert_eq!(response.stage_vote, Some(StageVote::Commit)); + } + + #[test] + fn static_stage_bulk_delete_without_prediction_votes_participant_error() { + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + + let response = stage_at(&mut core, 34, &[bulk_delete_plan("orders", None)]); + + assert_eq!(response.status, Status::Error); + assert_eq!(response.stage_vote, Some(StageVote::ParticipantError)); + } + + /// A materialized-sum source row whose join value the plan's resolution + /// does not cover votes `PredictionDrift`. A covered one stages. + #[test] + fn static_stage_bulk_delete_with_uncovered_sum_target_votes_prediction_drift() { + use crate::data::executor::handlers::transaction::redo_apply::calvin_fold_tests::{ + SOURCE, TARGET, configure_sum, + }; + let source_row = |core: &mut CoreLoop, surrogate: u32, account: &str| { + let body = doc_format::encode_to_msgpack( + &serde_json::json!({"account_id": account, "amount": 5}), + ); + core.sparse + .put( + DatabaseId::DEFAULT.as_u64(), + 1, + SOURCE, + &nodedb_types::StorageKey::for_surrogate(Surrogate::new(surrogate)), + &body, + ) + .expect("seed source row"); + }; + let resolved_delete = |predicted: Vec| { + let mut plan = bulk_delete_plan(SOURCE, Some(predicted)); + if let PhysicalPlan::Document( + nodedb_physical::physical_plan::DocumentOp::BulkDelete { + resolved_sum_targets, + .. + }, + ) = &mut plan + { + *resolved_sum_targets = + vec![nodedb_physical::physical_plan::ResolvedSumTarget::new( + TARGET, + "a1", + Surrogate::new(4242), + )]; + } + plan + }; + + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + configure_sum(&mut core); + source_row(&mut core, 5, "a2"); + let drifted = stage_at(&mut core, 35, &[resolved_delete(vec![5])]); + assert_drift_stages_nothing(&core, &drifted, 35); + + let dir = tempfile::tempdir().unwrap(); + let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); + configure_sum(&mut core); + source_row(&mut core, 6, "a1"); + let covered = stage_at(&mut core, 36, &[resolved_delete(vec![6])]); + assert_eq!(covered.status, Status::Ok, "{:?}", covered.error_code); + } + #[test] fn calvin_execute_static_stages_point_insert_into_overlay() { let dir = tempfile::tempdir().unwrap(); diff --git a/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs b/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs index bff0e14e4..f5389f7e1 100644 --- a/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs +++ b/nodedb/src/data/executor/handlers/control/calvin/test_commit.rs @@ -9,11 +9,11 @@ use nodedb_physical::physical_plan::{ }; use crate::bridge::envelope::{Response, Status}; -use crate::control::wal_replication::transaction_redo::collections::written_collections; use crate::control::wal_replication::transaction_redo::sum_targets::redo_sum_targets; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::task::ExecutionTask; use crate::types::{Lsn, TenantId}; +use nodedb_physical::physical_plan::written_collections; use super::shared::CalvinExecCtx; diff --git a/nodedb/src/data/executor/handlers/control/calvin_active_verify.rs b/nodedb/src/data/executor/handlers/control/calvin_active_verify.rs deleted file mode 100644 index 72ca1d437..000000000 --- a/nodedb/src/data/executor/handlers/control/calvin_active_verify.rs +++ /dev/null @@ -1,129 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Stage-time OLLP predicate verification for the dependent-read ACTIVE Calvin -//! path, which carries no versioned read-set. The leader-only `actual != -//! predicted` re-check must run HERE, before staging — at install time a -//! mismatch is swallowed as a degraded shard instead. Mirrors the -//! `BulkDelete`/`BulkUpdate` arms of [`CoreLoop::stage_calvin_overlay`]. - -use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan}; - -use crate::bridge::scan_filter::ScanFilter; -use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::handlers::bulk_dml::scan::{ - ollp_edges_match, ollp_predicted_doc_ids, ollp_surrogates_match, -}; -use crate::data::executor::task::ExecutionTask; - -impl CoreLoop { - /// OLLP verification of every predicate-DML plan in a dependent-read - /// ACTIVE Calvin txn, before staging. Only the data-group leader stages. `Ok(false)` means a - /// prediction drifted; caller returns `OllpRetryRequired`, stages nothing. - pub(in crate::data::executor) fn verify_calvin_active_ollp( - &self, - task: &ExecutionTask, - tid: u64, - plans: &[PhysicalPlan], - ) -> crate::Result { - let database_id = task.request.database_id.as_u64(); - for plan in plans { - // Only the document engine carries an OLLP prediction; exhaustive - // so a new `PhysicalPlan` variant forces a decision here. - let document_op = match plan { - PhysicalPlan::Document(op) => op, - PhysicalPlan::Vector(_) - | PhysicalPlan::Graph(_) - | PhysicalPlan::Kv(_) - | PhysicalPlan::Text(_) - | PhysicalPlan::Columnar(_) - | PhysicalPlan::Timeseries(_) - | PhysicalPlan::Spatial(_) - | PhysicalPlan::Crdt(_) - | PhysicalPlan::Query(_) - | PhysicalPlan::Meta(_) - | PhysicalPlan::Array(_) - | PhysicalPlan::ClusterArray(_) - | PhysicalPlan::ClusterEvent(_) => continue, - }; - let (collection, filter_bytes, predicted_surrogates, predicted_edges) = - match document_op { - DocumentOp::BulkDelete { - collection, - filters, - ollp_predicted_surrogates, - ollp_predicted_edges, - .. - } - | DocumentOp::BulkUpdate { - collection, - filters, - ollp_predicted_surrogates, - ollp_predicted_edges, - .. - } => ( - collection, - filters, - ollp_predicted_surrogates, - ollp_predicted_edges, - ), - // Mirrors `stage_calvin_overlay`'s non-bulk arms: no OLLP - // prediction to verify. - DocumentOp::PointGet { .. } - | DocumentOp::PointPut { .. } - | DocumentOp::PointInsert { .. } - | DocumentOp::PointDelete { .. } - | DocumentOp::PointUpdate { .. } - | DocumentOp::Upsert { .. } - | DocumentOp::BatchUpsert { .. } - | DocumentOp::BatchInsert { .. } - | DocumentOp::Scan { .. } - | DocumentOp::RangeScan { .. } - | DocumentOp::Register { .. } - | DocumentOp::IndexLookup { .. } - | DocumentOp::IndexedFetch { .. } - | DocumentOp::DropIndex { .. } - | DocumentOp::BackfillIndex { .. } - | DocumentOp::Truncate { .. } - | DocumentOp::EstimateCount { .. } - | DocumentOp::InsertSelect { .. } - | DocumentOp::UpdateFromJoin { .. } - | DocumentOp::Merge { .. } - | DocumentOp::ResolveWrite(_) - | DocumentOp::MaterializeScan { .. } - // Carries decided rows, not a predicate to re-verify: each - // mutation's own content precondition is the drift check. - | DocumentOp::ResolvedWrite { .. } - | DocumentOp::ApplyBalanceDelta { .. } => continue, - }; - let Some(predicted) = predicted_surrogates.as_deref() else { - // A bulk op with no prediction is the non-OLLP (static-set) - // shape; `stage_calvin_overlay` rejects it loudly at staging. - continue; - }; - - let filters: Vec = if filter_bytes.is_empty() { - Vec::new() - } else { - zerompk::from_msgpack(filter_bytes).map_err(|e| crate::Error::Serialization { - format: "msgpack".to_string(), - detail: format!("calvin active ollp verify: deserialize filters: {e}"), - })? - }; - let matching_ids = - self.scan_matching_documents(database_id, tid, collection.as_str(), &filters)?; - - if !ollp_surrogates_match(&matching_ids, predicted) { - return Ok(false); - } - if let Some(predicted_edges) = predicted_edges.as_deref() { - let apply_ids = ollp_predicted_doc_ids(predicted); - let actual = - self.ollp_actual_edges(database_id, tid, collection.as_str(), &apply_ids); - if !ollp_edges_match(actual, predicted_edges) { - return Ok(false); - } - } - } - Ok(true) - } -} diff --git a/nodedb/src/data/executor/handlers/control/calvin_overlay_stage.rs b/nodedb/src/data/executor/handlers/control/calvin_overlay_stage.rs index bca1906f2..238978ac7 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_overlay_stage.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_overlay_stage.rs @@ -205,44 +205,9 @@ impl CoreLoop { } affected_reply(documents.len()).map_err(ErrorCode::from) } - PhysicalPlan::Document(DocumentOp::BulkDelete { - collection, - ollp_predicted_surrogates, - rls_write_check, - declared_primary_key, - .. - }) => self - .stage_calvin_bulk_delete(super::calvin_overlay_stage_bulk::CalvinBulkDeleteStage { - task, - tid, - txn_id, - collection: collection.as_str(), - ollp_predicted_surrogates: ollp_predicted_surrogates.as_deref(), - rls_write_check, - declared_primary_key: declared_primary_key.as_deref(), - }) - .and_then(affected_reply) - .map_err(ErrorCode::from), - PhysicalPlan::Document(DocumentOp::BulkUpdate { - collection, - updates, - ollp_predicted_surrogates, - rls_write_check, - declared_primary_key, - .. - }) => self - .stage_calvin_bulk_update(super::calvin_overlay_stage_bulk::CalvinBulkUpdateStage { - task, - tid, - txn_id, - collection: collection.as_str(), - updates, - ollp_predicted_surrogates: ollp_predicted_surrogates.as_deref(), - rls_write_check, - declared_primary_key: declared_primary_key.as_deref(), - }) - .and_then(affected_reply) - .map_err(ErrorCode::from), + PhysicalPlan::Document( + op @ (DocumentOp::BulkDelete { .. } | DocumentOp::BulkUpdate { .. }), + ) => self.stage_calvin_bulk_plan(task, tid, txn_id, op), PhysicalPlan::Document(DocumentOp::BatchInsert { collection, documents, diff --git a/nodedb/src/data/executor/handlers/control/calvin_overlay_stage_bulk.rs b/nodedb/src/data/executor/handlers/control/calvin_overlay_stage_bulk.rs index 1bd0d93ff..79440320d 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_overlay_stage_bulk.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_overlay_stage_bulk.rs @@ -7,14 +7,19 @@ //! # The determinism rule //! //! Staging resolves EXACTLY the CP-injected `ollp_predicted_surrogates` set, -//! verbatim, on every replica, via the `ollp_predicted_doc_ids` primitive. A -//! live predicate rescan is NOT used as the row set, because a follower's -//! local snapshot can legitimately lag the leader's verified prediction -//! window. Re-deriving the row set locally would diverge across replicas. -//! The install applies the redo record `CalvinResolve` builds from this -//! staging, so the staged rows are the rows the install writes. The -//! `stage_bulk_delete` / `stage_bulk_update` session-transaction handlers do -//! a live rescan and are NOT reused here for this reason. +//! verbatim, via the `ollp_predicted_doc_ids` primitive. The install applies +//! the redo record `CalvinResolve` builds from this staging, so the staged +//! rows are the rows the install writes. +//! +//! # The prediction check +//! +//! Before a bulk plan stages, the leader checks its prediction against the +//! rows the predicate matches at the plan's turn: BASE with the overlay of +//! the plans staged before it. The surrogate set, the implicit edges of +//! matched edge documents, and the materialized-sum coverage must all hold. +//! A miss refuses the stage with `OllpRetryRequired`, which votes +//! `PredictionDrift`. Only the data-group leader stages, and a bulk plan +//! holds its collection's lock, so a lagging follower never judges. //! //! Reading each predicted surrogate's CURRENT body (for `BulkUpdate`'s //! post-image and read-your-own-writes) is still safe to source from local @@ -31,11 +36,14 @@ //! Each stage returns the number of rows it staged, the affected count the //! statement reports. -use nodedb_physical::physical_plan::UpdateValue; +use nodedb_physical::physical_plan::{ResolvedSumTarget, UpdateValue}; use nodedb_types::Surrogate; +use crate::bridge::envelope::ErrorCode; use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::enforcement::materialized_sum::divergence::SumTargetCheck; use crate::data::executor::handlers::bulk_dml::scan::ollp_predicted_doc_ids; +use crate::data::executor::handlers::bulk_dml::{TxnPrediction, decode_bulk_filters}; use crate::data::executor::handlers::transaction::overlay::Staged; use crate::data::executor::handlers::transaction::stage_write::{ StagedStatement, stored_row_identity, @@ -43,6 +51,17 @@ use crate::data::executor::handlers::transaction::stage_write::{ use crate::data::executor::task::ExecutionTask; use crate::types::{DatabaseId, TenantId, TxnId}; +/// What a Calvin bulk plan carries for its prediction check. +#[derive(Clone, Copy)] +pub(in crate::data::executor) struct CalvinBulkCheck<'a> { + /// The predicate's serialized filters. + pub filters: &'a [u8], + /// The rows the Control Plane predicted. `None` when the plan carries none. + pub prediction: Option>, + /// The materialized-sum resolution the plan carries. + pub resolved_sum_targets: &'a [ResolvedSumTarget], +} + /// Borrowed inputs for [`CoreLoop::stage_calvin_bulk_delete`], grouped so the /// method stays within the argument-count limit. pub(in crate::data::executor) struct CalvinBulkDeleteStage<'a> { @@ -50,7 +69,7 @@ pub(in crate::data::executor) struct CalvinBulkDeleteStage<'a> { pub tid: u64, pub txn_id: TxnId, pub collection: &'a str, - pub ollp_predicted_surrogates: Option<&'a [u32]>, + pub check: CalvinBulkCheck<'a>, /// Compiled RLS write policy deciding each removed row's pre-image. pub rls_write_check: &'a nodedb_types::RlsWriteCheck, /// The collection's DDL-declared primary key, when it has one. Names the @@ -81,7 +100,7 @@ pub(in crate::data::executor) struct CalvinBulkUpdateStage<'a> { pub txn_id: TxnId, pub collection: &'a str, pub updates: &'a [(String, UpdateValue)], - pub ollp_predicted_surrogates: Option<&'a [u32]>, + pub check: CalvinBulkCheck<'a>, /// Compiled RLS write policy gating each staged post-image. pub rls_write_check: &'a nodedb_types::RlsWriteCheck, /// The collection's DDL-declared primary key, when it has one. A staged @@ -104,16 +123,18 @@ impl CoreLoop { tid, txn_id, collection, - ollp_predicted_surrogates, + check, rls_write_check, declared_primary_key, } = params; - let Some(predicted) = ollp_predicted_surrogates else { + let Some(prediction) = check.prediction else { return Err(missing_prediction_error(collection)); }; let database_id = task.request.database_id; let coll_key: (DatabaseId, TenantId, String) = (database_id, TenantId::new(tid), collection.to_string()); + self.check_calvin_prediction(txn_id, &coll_key, &[], &check, prediction)?; + let predicted = prediction.surrogates; let mut predicted_sorted: Vec = predicted.to_vec(); predicted_sorted.sort_unstable(); @@ -125,7 +146,6 @@ impl CoreLoop { // body removes nothing; its tombstone is keyed by the decimal // surrogate. let strict_schema = self.resolve_strict_schema(database_id.as_u64(), tid, collection); - let bitemporal = self.is_bitemporal(database_id.as_u64(), tid, collection); let mut rows: Vec<(u32, nodedb_types::RowIdentity, Option>)> = Vec::with_capacity(doc_ids.len()); for (surrogate, doc_id) in predicted_sorted.into_iter().zip(doc_ids) { @@ -134,7 +154,6 @@ impl CoreLoop { coll_key: &coll_key, surrogate, storage_key: &doc_id, - bitemporal, })?; let identity = match &body { Some(body) => { @@ -199,17 +218,18 @@ impl CoreLoop { txn_id, collection, updates, - ollp_predicted_surrogates, + check, rls_write_check, declared_primary_key, } = params; - let Some(predicted) = ollp_predicted_surrogates else { + let Some(prediction) = check.prediction else { return Err(missing_prediction_error(collection)); }; let database_id = task.request.database_id; let coll_key: (DatabaseId, TenantId, String) = (database_id, TenantId::new(tid), collection.to_string()); - let bitemporal = self.is_bitemporal(database_id.as_u64(), tid, collection); + self.check_calvin_prediction(txn_id, &coll_key, updates, &check, prediction)?; + let predicted = prediction.surrogates; let mut predicted_sorted: Vec = predicted.to_vec(); predicted_sorted.sort_unstable(); @@ -226,7 +246,6 @@ impl CoreLoop { coll_key: &coll_key, surrogate, storage_key: &storage_key, - bitemporal, })? else { continue; @@ -281,6 +300,46 @@ impl CoreLoop { } Ok(updated) } + + /// Refuse a Calvin bulk plan whose prediction does not hold at its turn + /// in the transaction: BASE with the overlay of the plans staged before + /// it. The surrogate set and the implicit edges must equal the rows'. The + /// materialized-sum resolution must cover every join value the rows + /// require, read after `updates` for an update. A miss refuses with + /// `OllpRetryRequired`, which votes `PredictionDrift`. + fn check_calvin_prediction( + &self, + txn_id: TxnId, + coll_key: &(DatabaseId, TenantId, String), + updates: &[(String, UpdateValue)], + check: &CalvinBulkCheck<'_>, + prediction: TxnPrediction<'_>, + ) -> crate::Result<()> { + let drift = || crate::Error::DataPlane(ErrorCode::OllpRetryRequired); + let filters = decode_bulk_filters(check.filters)?; + let rows = self.txn_matching_rows(txn_id, coll_key, &filters)?; + if !prediction.holds(&rows) { + return Err(drift()); + } + let sum_check = SumTargetCheck { + database_id: coll_key.0.as_u64(), + tid: coll_key.1.as_u64(), + collection: coll_key.2.as_str(), + updates, + resolved: check.resolved_sum_targets, + }; + if !self.declares_materialized_sums(&sum_check) { + return Ok(()); + } + let images: Vec = rows + .iter() + .filter_map(|(_, body)| self.decode_source_row(&sum_check, body)) + .collect(); + if self.sum_targets_diverged(&sum_check, &images) { + return Err(drift()); + } + Ok(()) + } } /// One row a Calvin bulk stage reads under BASE ∪ OVERLAY. @@ -289,7 +348,6 @@ struct CalvinRowRead<'a> { coll_key: &'a (DatabaseId, TenantId, String), surrogate: u32, storage_key: &'a nodedb_types::StorageKey, - bitemporal: bool, } impl CoreLoop { @@ -308,20 +366,11 @@ impl CoreLoop { return Ok(None); } let (database_id, tenant, collection) = read.coll_key; - if read.bitemporal { - self.sparse.versioned_get_current( - database_id.as_u64(), - tenant.as_u64(), - collection, - read.storage_key, - ) - } else { - self.sparse.get( - database_id.as_u64(), - tenant.as_u64(), - collection, - read.storage_key, - ) - } + self.current_row_body( + database_id.as_u64(), + tenant.as_u64(), + collection, + read.storage_key, + ) } } diff --git a/nodedb/src/data/executor/handlers/control/calvin_overlay_stage_bulk_dispatch.rs b/nodedb/src/data/executor/handlers/control/calvin_overlay_stage_bulk_dispatch.rs new file mode 100644 index 000000000..7627b56cd --- /dev/null +++ b/nodedb/src/data/executor/handlers/control/calvin_overlay_stage_bulk_dispatch.rs @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Dispatch of one Calvin predicate plan (`DocumentOp::BulkDelete` / +//! `BulkUpdate`) to its stage in +//! [`calvin_overlay_stage_bulk`][super::calvin_overlay_stage_bulk], with the +//! prediction the plan carries. + +use nodedb_physical::physical_plan::DocumentOp; + +use super::calvin_overlay_stage_bulk::{ + CalvinBulkCheck, CalvinBulkDeleteStage, CalvinBulkUpdateStage, +}; +use crate::bridge::envelope::ErrorCode; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::bulk_dml::TxnPrediction; +use crate::data::executor::response_codec; +use crate::data::executor::task::ExecutionTask; +use crate::types::TxnId; + +impl CoreLoop { + /// Stage one Calvin predicate plan and answer its `{"affected": n}` + /// reply. Any other document op is refused: it never routes here. + pub(in crate::data::executor) fn stage_calvin_bulk_plan( + &mut self, + task: &ExecutionTask, + tid: u64, + txn_id: TxnId, + op: &DocumentOp, + ) -> Result, ErrorCode> { + let staged = match op { + DocumentOp::BulkDelete { + collection, + filters, + ollp_predicted_surrogates, + ollp_predicted_edges, + rls_write_check, + resolved_sum_targets, + declared_primary_key, + .. + } => self.stage_calvin_bulk_delete(CalvinBulkDeleteStage { + task, + tid, + txn_id, + collection: collection.as_str(), + check: CalvinBulkCheck { + filters, + prediction: TxnPrediction::of(ollp_predicted_surrogates, ollp_predicted_edges), + resolved_sum_targets, + }, + rls_write_check, + declared_primary_key: declared_primary_key.as_deref(), + }), + DocumentOp::BulkUpdate { + collection, + filters, + updates, + ollp_predicted_surrogates, + ollp_predicted_edges, + rls_write_check, + resolved_sum_targets, + declared_primary_key, + .. + } => self.stage_calvin_bulk_update(CalvinBulkUpdateStage { + task, + tid, + txn_id, + collection: collection.as_str(), + updates, + check: CalvinBulkCheck { + filters, + prediction: TxnPrediction::of(ollp_predicted_surrogates, ollp_predicted_edges), + resolved_sum_targets, + }, + rls_write_check, + declared_primary_key: declared_primary_key.as_deref(), + }), + DocumentOp::PointGet { .. } + | DocumentOp::PointPut { .. } + | DocumentOp::PointInsert { .. } + | DocumentOp::PointDelete { .. } + | DocumentOp::PointUpdate { .. } + | DocumentOp::Upsert { .. } + | DocumentOp::BatchUpsert { .. } + | DocumentOp::BatchInsert { .. } + | DocumentOp::Scan { .. } + | DocumentOp::RangeScan { .. } + | DocumentOp::Register { .. } + | DocumentOp::IndexLookup { .. } + | DocumentOp::IndexedFetch { .. } + | DocumentOp::DropIndex { .. } + | DocumentOp::BackfillIndex { .. } + | DocumentOp::Truncate { .. } + | DocumentOp::EstimateCount { .. } + | DocumentOp::InsertSelect { .. } + | DocumentOp::UpdateFromJoin { .. } + | DocumentOp::Merge { .. } + | DocumentOp::ResolveWrite(_) + | DocumentOp::MaterializeScan { .. } + | DocumentOp::ResolvedWrite { .. } + | DocumentOp::ApplyBalanceDelta { .. } => { + return Err(ErrorCode::Internal { + detail: "a document op that is not a predicate write reached Calvin \ + predicate staging" + .into(), + }); + } + }; + staged + .and_then(|affected| response_codec::encode_count("affected", affected)) + .map_err(ErrorCode::from) + } +} diff --git a/nodedb/src/data/executor/handlers/control/calvin_passive_read.rs b/nodedb/src/data/executor/handlers/control/calvin_passive_read.rs index 831bd828d..6f511ef4f 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_passive_read.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_passive_read.rs @@ -44,9 +44,10 @@ impl CoreLoop { } => surrogates .iter() .map(|&surrogate| { + // The row at its current version, so a bitemporal + // collection reads too. let stored = self - .sparse - .get( + .current_row_body( database_id, tid, collection, diff --git a/nodedb/src/data/executor/handlers/control/calvin_reply/images.rs b/nodedb/src/data/executor/handlers/control/calvin_reply/images.rs index 9b9d7a4c7..f568d8560 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_reply/images.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_reply/images.rs @@ -88,13 +88,8 @@ impl CoreLoop { match engine { RowEngine::Document | RowEngine::Crdt => { let key = StorageKey::for_surrogate(surrogate); - let read = if self.is_bitemporal(database_id, tid, collection) { - self.sparse - .versioned_get_current(database_id, tid, collection, &key) - } else { - self.sparse.get(database_id, tid, collection, &key) - }; - read.map_err(ErrorCode::from) + self.current_row_body(database_id, tid, collection, &key) + .map_err(ErrorCode::from) } RowEngine::Kv => { let key = kv_raw_key(identity)?; diff --git a/nodedb/src/data/executor/handlers/control/calvin_resolve.rs b/nodedb/src/data/executor/handlers/control/calvin_resolve.rs index 52c16f3f2..fe5e90aa5 100644 --- a/nodedb/src/data/executor/handlers/control/calvin_resolve.rs +++ b/nodedb/src/data/executor/handlers/control/calvin_resolve.rs @@ -372,7 +372,7 @@ mod tests { let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); let task = make_task(); - // Seed 3 rows; the predicted set names a known 2-row subset. + // Seed 3 rows; the empty predicate matches all three, as predicted. seed_row(&mut core, "orders", 10, "a", "1"); seed_row(&mut core, "orders", 11, "a", "1"); seed_row(&mut core, "orders", 12, "a", "1"); @@ -382,7 +382,7 @@ mod tests { &task, 3, 0, - &[bulk_delete_plan("orders", Some(vec![10, 12]))], + &[bulk_delete_plan("orders", Some(vec![10, 11, 12]))], ); let resp = core.execute_calvin_resolve(&task, 3, 0); @@ -390,46 +390,50 @@ mod tests { let record = resolved_record(&resp); assert_eq!( record.ops.len(), - 2, + 3, "exactly one Delete sub-record per predicted surrogate" ); - assert_eq!(deleted_surrogates(&record), vec![10, 12]); + assert_eq!(deleted_surrogates(&record), vec![10, 11, 12]); } #[test] - fn calvin_bulk_delete_ignores_drift_from_predicted_set() { + fn calvin_bulk_delete_refuses_drift_from_predicted_set() { let dir = tempfile::tempdir().unwrap(); let (mut core, _tx, _rx) = make_core_with_dir(dir.path()); let task = make_task(); - // 3 rows exist in base and would ALL match an empty-predicate - // (match-everything) live rescan. The predicted set names only 2 of - // them -- if staging re-derived the row set via a live scan instead - // of trusting the predicted set, the redo would carry 3 deletes, not - // 2. This is the determinism proof: staging must key off the - // predicted set the install applies, not a fresh predicate scan. + // 3 rows match the empty (match-everything) predicate, but the plan + // predicts only 2 of them. The stage refuses the drifted prediction + // and stages nothing, so no stale redo exists to resolve. seed_row(&mut core, "orders", 20, "a", "1"); seed_row(&mut core, "orders", 21, "a", "1"); seed_row(&mut core, "orders", 22, "a", "1"); - stage( - &mut core, + let ctx = CalvinExecCtx { + epoch: 4, + position: 0, + epoch_system_ms: 0, + }; + let staged = core.execute_calvin_execute_static( &task, - 4, - 0, + ctx, + &TenantId::new(1), &[bulk_delete_plan("orders", Some(vec![20, 21]))], + &[], + &[], + ); + assert_eq!(staged.status, Status::Error); + assert_eq!( + staged.stage_vote, + Some(crate::bridge::envelope::StageVote::PredictionDrift) ); let resp = core.execute_calvin_resolve(&task, 4, 0); - assert_eq!(resp.status, Status::Ok, "resolve must succeed: {resp:?}"); - let record = resolved_record(&resp); assert_eq!( - record.ops.len(), - 2, - "redo must reflect ONLY the predicted surrogates, not the live \ - (drifted) match-everything set" + resp.status, + Status::Error, + "a refused stage leaves nothing to resolve" ); - assert_eq!(deleted_surrogates(&record), vec![20, 21]); } #[test] diff --git a/nodedb/src/data/executor/handlers/control/crdt_doc.rs b/nodedb/src/data/executor/handlers/control/crdt_doc.rs index df71db572..a73fe026d 100644 --- a/nodedb/src/data/executor/handlers/control/crdt_doc.rs +++ b/nodedb/src/data/executor/handlers/control/crdt_doc.rs @@ -267,7 +267,8 @@ impl CoreLoop { if UpdateTransitionRules::of(&config.enforcement).is_empty() { return Ok(None); } - self.sparse.get( + // The row at its current version, as the materialize writes it. + self.current_row_body( task.request.database_id.as_u64(), task.request.tenant_id.as_u64(), collection, diff --git a/nodedb/src/data/executor/handlers/control/mod.rs b/nodedb/src/data/executor/handlers/control/mod.rs index 5a2284b64..20a857cb2 100644 --- a/nodedb/src/data/executor/handlers/control/mod.rs +++ b/nodedb/src/data/executor/handlers/control/mod.rs @@ -3,9 +3,9 @@ //! Control operation handlers — module root. pub mod calvin; -mod calvin_active_verify; mod calvin_overlay_stage; mod calvin_overlay_stage_bulk; +mod calvin_overlay_stage_bulk_dispatch; mod calvin_passive_read; pub mod calvin_reply; mod calvin_resolve; diff --git a/nodedb/src/data/executor/handlers/convert.rs b/nodedb/src/data/executor/handlers/convert.rs index 0754a48c7..f7815dc71 100644 --- a/nodedb/src/data/executor/handlers/convert.rs +++ b/nodedb/src/data/executor/handlers/convert.rs @@ -67,6 +67,21 @@ impl CoreLoop { let source_format = source_format_of(source_storage_mode); + // A bitemporal collection holds its rows only as versions. Each + // version is rewritten in place, so its history survives. + if self.is_bitemporal(task.request.database_id.as_u64(), tid, collection) { + return self.convert_bitemporal_collection( + task, + super::convert_bitemporal::BitemporalConvert { + tid, + collection, + target_type, + schema_json, + source_format, + }, + ); + } + match target_type { "document_strict" => { self.convert_to_strict(task, tid, collection, schema_json, source_format) @@ -99,48 +114,18 @@ impl CoreLoop { schema_json: &str, source_format: SparseBodyFormat, ) -> Response { - // Parse the target schema from JSON column definitions. - let columns: Vec = match sonic_rs::from_str(schema_json) { - Ok(c) => c, - Err(e) => { - return self.response_error( - task, - ErrorCode::Internal { - detail: format!("invalid schema JSON: {e}"), - }, - ); - } + let columns = match parse_target_columns(schema_json) { + Ok(columns) => columns, + Err(code) => return self.response_error(task, code), }; - - if columns.is_empty() { - return self.response_error( - task, - ErrorCode::Internal { - detail: "schema must have at least one column".into(), - }, - ); - } - let schema = StrictSchema { columns, version: 1, dropped_columns: Vec::new(), bitemporal: false, }; - let declared_primary_key = schema - .columns - .iter() - .find(|c| c.primary_key) - .map(|c| c.name.as_str()); - // A system row id keys a schema with no declared key: it is an integer - // the write path fills from the surrogate. Every other key column holds - // the client-visible identity text. - let identity_column = match declared_primary_key { - Some(key) if key == nodedb_types::ROWID_COLUMN => nodedb_types::ROWID_COLUMN, - other => { - nodedb_types::declared_key(other).unwrap_or(nodedb_types::DEFAULT_IDENTITY_COLUMN) - } - }; + let declared_primary_key = target_primary_key(&schema); + let identity_column = target_identity_column(declared_primary_key); // Scan all existing documents. let database_id = task.request.database_id.as_u64(); @@ -158,30 +143,7 @@ impl CoreLoop { for (doc_id, doc_bytes) in &docs { let normalized = sparse_body_to_msgpack(doc_bytes, source_format.as_format_ref()); - // A row that lacks the target's key column has no client-visible - // identity yet: inject the surrogate's decimal string there so the - // target schema's NOT NULL key column has something to validate. - let synth_id = doc_id.to_identity(); - let with_id = if identity_column == nodedb_types::ROWID_COLUMN { - // A row that carries its system row id keeps it. One that - // lacks it takes the surrogate, as a write fills it. - if msgpack_scan::extract_field(&normalized, 0, nodedb_types::ROWID_COLUMN).is_some() - { - normalized.to_vec() - } else { - let mut cell = Vec::with_capacity(9); - msgpack_scan::writer::write_i64( - &mut cell, - i64::from(doc_id.surrogate().as_u32()), - ); - msgpack_scan::merge_fields( - &normalized, - &[(nodedb_types::ROWID_COLUMN, cell.as_slice())], - ) - } - } else { - msgpack_scan::inject_str_field(&normalized, identity_column, synth_id.as_str()) - }; + let with_id = with_target_identity(&normalized, identity_column, doc_id); // The identity a user recognizes: the declared primary key's value, // or `id`, read from the row itself — never the internal surrogate. let identity = RowIdentity::of_stored_row(&normalized, declared_primary_key, *doc_id); @@ -336,6 +298,65 @@ impl CoreLoop { } } +/// The target strict columns a CONVERT carries as JSON. An empty or +/// unparsable list is an internal error: the Control Plane built it. +pub(super) fn parse_target_columns(schema_json: &str) -> Result, ErrorCode> { + let columns: Vec = + sonic_rs::from_str(schema_json).map_err(|e| ErrorCode::Internal { + detail: format!("invalid schema JSON: {e}"), + })?; + if columns.is_empty() { + return Err(ErrorCode::Internal { + detail: "schema must have at least one column".into(), + }); + } + Ok(columns) +} + +/// The target schema's declared primary-key column, if it declares one. +pub(super) fn target_primary_key(schema: &StrictSchema) -> Option<&str> { + schema + .columns + .iter() + .find(|c| c.primary_key) + .map(|c| c.name.as_str()) +} + +/// The column a converted row's identity lands in. +/// +/// A system row id keys a schema with no declared key: it is an integer the +/// write path fills from the surrogate. Every other key column holds the +/// client-visible identity text. +pub(super) fn target_identity_column(declared_primary_key: Option<&str>) -> &str { + match declared_primary_key { + Some(key) if key == nodedb_types::ROWID_COLUMN => nodedb_types::ROWID_COLUMN, + other => nodedb_types::declared_key(other).unwrap_or(nodedb_types::DEFAULT_IDENTITY_COLUMN), + } +} + +/// `normalized` with its identity under `identity_column`. +/// +/// A row that lacks the target's key column has no client-visible identity +/// yet: the surrogate's decimal string fills it, so the target schema's NOT +/// NULL key column has something to validate. A row that carries its system +/// row id keeps it. One that lacks it takes the surrogate, as a write fills it. +pub(super) fn with_target_identity( + normalized: &[u8], + identity_column: &str, + doc_id: &nodedb_types::StorageKey, +) -> Vec { + if identity_column != nodedb_types::ROWID_COLUMN { + let synth_id = doc_id.to_identity(); + return msgpack_scan::inject_str_field(normalized, identity_column, synth_id.as_str()); + } + if msgpack_scan::extract_field(normalized, 0, nodedb_types::ROWID_COLUMN).is_some() { + return normalized.to_vec(); + } + let mut cell = Vec::with_capacity(9); + msgpack_scan::writer::write_i64(&mut cell, i64::from(doc_id.surrogate().as_u32())); + msgpack_scan::merge_fields(normalized, &[(nodedb_types::ROWID_COLUMN, cell.as_slice())]) +} + /// Where one converted row lands. struct ConvertedRow<'a> { database_id: u64, diff --git a/nodedb/src/data/executor/handlers/convert_bitemporal.rs b/nodedb/src/data/executor/handlers/convert_bitemporal.rs new file mode 100644 index 000000000..a16d14597 --- /dev/null +++ b/nodedb/src/data/executor/handlers/convert_bitemporal.rs @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! CONVERT COLLECTION on a bitemporal collection. +//! +//! A bitemporal collection holds its rows only as versions, and its history +//! is append-only. The conversion rewrites every live version in place, in +//! the target encoding, under its own system time and valid interval. +//! Tombstone versions stay as they are. The current read and every +//! `AS OF SYSTEM TIME` read then return the same rows as before. +//! +//! A schemaless or strict document target holds versions. Any other target +//! is refused before a version moves. + +use std::collections::{HashMap, HashSet}; + +use nodedb_types::StorageKey; +use nodedb_types::columnar::StrictSchema; + +use super::convert::{ + parse_target_columns, target_identity_column, target_primary_key, with_target_identity, +}; +use crate::bridge::envelope::{ErrorCode, Response}; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::response_codec; +use crate::data::executor::scan_normalize::sparse_body_to_msgpack; +use crate::data::executor::sparse_body_format::SparseBodyFormat; +use crate::data::executor::strict_format; +use crate::data::executor::task::ExecutionTask; +use crate::engine::sparse::btree_versioned::{VersionedPut, VersionedScanParams}; + +/// Inputs to [`CoreLoop::convert_bitemporal_collection`]. +pub(in crate::data::executor) struct BitemporalConvert<'a> { + pub tid: u64, + pub collection: &'a str, + pub target_type: &'a str, + /// The target strict columns as JSON. Empty for a schemaless target. + pub schema_json: &'a str, + /// How the source versions are encoded. + pub source_format: SparseBodyFormat, +} + +/// The encoding a version takes in the target. +enum VersionTarget { + /// Schemaless MessagePack. + Schemaless, + /// A bitemporal strict schema, with the column a row's identity lands in. + Strict { + schema: StrictSchema, + identity_column: String, + }, +} + +impl CoreLoop { + /// Convert a bitemporal collection: rewrite every live version in the + /// target encoding. Returns `{"converted": N}`, where N counts the live + /// rows. + pub(in crate::data::executor) fn convert_bitemporal_collection( + &mut self, + task: &ExecutionTask, + c: BitemporalConvert<'_>, + ) -> Response { + let converted = version_target(c.target_type, c.schema_json) + .and_then(|target| self.rewrite_versions(task, &c, &target)); + let converted = match converted { + Ok(converted) => converted, + Err(code) => return self.response_error(task, code), + }; + tracing::info!(collection = %c.collection, converted, "bitemporal collection converted"); + let result = serde_json::json!({ + "converted": converted, + "target_type": c.target_type, + "collection": c.collection, + }); + match response_codec::encode_json_as_msgpack(&result) { + Ok(payload) => self.response_with_payload(task, payload), + Err(e) => self.response_error(task, ErrorCode::from(e)), + } + } + + /// Rewrite every live version of the collection in `target`'s encoding, + /// in one transaction. The current version of each live row also + /// re-indexes its text and refreshes the document cache. + /// + /// Returns the number of live rows. + fn rewrite_versions( + &mut self, + task: &ExecutionTask, + c: &BitemporalConvert<'_>, + target: &VersionTarget, + ) -> Result { + let database_id = task.request.database_id.as_u64(); + let (tid, collection) = (c.tid, c.collection); + let versions = self.sparse.versioned_scan_all( + VersionedScanParams { + database_id, + tenant: tid, + coll: collection, + sys_cutoff_ms: None, + valid_at_ms: None, + limit: usize::MAX, + }, + &|_: &StorageKey, _: &[u8]| true, + &crate::engine::sparse::scan_stop::never_stop, + )?; + let live: HashSet = self + .current_rows(database_id, tid, collection, usize::MAX)? + .into_iter() + .map(|(key, _)| key) + .collect(); + // A live row reads its newest live version. + let mut current_version: HashMap = HashMap::new(); + for version in versions.iter().filter(|v| live.contains(&v.doc_id)) { + let newest = current_version + .entry(version.doc_id) + .or_insert(version.system_from_ms); + *newest = (*newest).max(version.system_from_ms); + } + // A schemaless source already stores what a schemaless target holds. + if matches!(target, VersionTarget::Schemaless) + && !matches!(c.source_format, SparseBodyFormat::Strict(_)) + { + return Ok(live.len() as u64); + } + + let txn = self.sparse.begin_write()?; + let mut current_bodies: Vec<(StorageKey, Vec)> = Vec::with_capacity(live.len()); + for version in &versions { + let msgpack = version_msgpack(&version.body, &c.source_format).ok_or_else(|| { + strict_format::undecodable_strict_row( + collection, + version.doc_id.to_identity().as_str(), + ) + })?; + // `visible` is the row as a read sees it: the text index follows it. + let (stored, visible) = match target { + VersionTarget::Schemaless => (msgpack.clone(), msgpack), + VersionTarget::Strict { + schema, + identity_column, + } => { + let with_id = with_target_identity(&msgpack, identity_column, &version.doc_id); + let tuple = strict_format::bytes_to_binary_tuple_bitemporal( + &with_id, + schema, + version.system_from_ms, + version.valid_from_ms, + version.valid_until_ms, + collection, + )?; + let visible = strict_format::binary_tuple_to_row_value(&tuple, schema) + .and_then(|row| nodedb_types::value_to_msgpack(&row).ok()) + .ok_or_else(|| { + strict_format::undecodable_strict_row( + collection, + version.doc_id.to_identity().as_str(), + ) + })?; + (tuple, visible) + } + }; + self.sparse.versioned_put_in_txn( + &txn, + VersionedPut { + database_id, + tenant: tid, + coll: collection, + doc_id: &version.doc_id, + sys_from_ms: version.system_from_ms, + valid_from_ms: version.valid_from_ms, + valid_until_ms: version.valid_until_ms, + body: &stored, + }, + )?; + if current_version.get(&version.doc_id) == Some(&version.system_from_ms) { + let new_doc = crate::data::executor::doc_format::decode_document(&visible)?; + self.update_reindex_text( + &txn, + super::point::update_reindex_text::UpdateTextReindex { + database_id, + tid, + collection, + surrogate: version.doc_id.surrogate(), + new_doc: &new_doc, + }, + )?; + current_bodies.push((version.doc_id, stored)); + } + } + txn.commit().map_err(|e| ErrorCode::Internal { + detail: format!("bitemporal convert commit: {e}"), + })?; + // Write-through: a point-get after this statement sees the converted + // current version, never a cache entry from before the conversion. + for (doc_id, body) in ¤t_bodies { + self.doc_cache + .put(database_id, tid, collection, doc_id, body); + } + Ok(live.len() as u64) + } +} + +/// The encoding a CONVERT target gives a bitemporal collection's versions. +/// +/// A key-value target holds no row versions, so it is refused with nothing +/// moved: converting would erase the collection's history. +fn version_target(target_type: &str, schema_json: &str) -> Result { + match target_type { + "document_schemaless" => Ok(VersionTarget::Schemaless), + "document_strict" => { + let columns = parse_target_columns(schema_json)?; + let schema = + StrictSchema::new_bitemporal(columns).map_err(|e| ErrorCode::DataException { + detail: format!("bitemporal strict target schema: {e}"), + })?; + let identity_column = target_identity_column(target_primary_key(&schema)).to_string(); + Ok(VersionTarget::Strict { + schema, + identity_column, + }) + } + other => Err(ErrorCode::Unsupported { + detail: format!( + "CONVERT of a bitemporal collection to '{other}': the target holds no row \ + versions, so the collection's history would be lost" + ), + }), + } +} + +/// A source version as MessagePack, without the reserved bitemporal columns +/// a strict tuple carries. `None` when a strict tuple does not decode. +fn version_msgpack(body: &[u8], source: &SparseBodyFormat) -> Option> { + match source { + SparseBodyFormat::Strict(schema) => { + let row = strict_format::binary_tuple_to_row_value(body, schema)?; + nodedb_types::value_to_msgpack(&row).ok() + } + SparseBodyFormat::Document | SparseBodyFormat::VectorSidecar => { + Some(sparse_body_to_msgpack(body, source.as_format_ref()).into_owned()) + } + } +} diff --git a/nodedb/src/data/executor/handlers/document/index_backfill_versions.rs b/nodedb/src/data/executor/handlers/document/index_backfill_versions.rs new file mode 100644 index 000000000..04c2c147e --- /dev/null +++ b/nodedb/src/data/executor/handlers/document/index_backfill_versions.rs @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! CREATE INDEX backfill of a bitemporal collection's versioned index. +//! +//! The versioned index answers a lookup at a system-time cutoff, so it holds +//! an entry for each value at the system time the value became current. The +//! backfill walks every version of every row in ascending system time and +//! writes what the write path writes at that version's time: +//! +//! - A live version asserts each of its values live. +//! - It tombstones each value the previous version held and it drops. +//! - A tombstone version tombstones every value the previous version held. +//! +//! An `AS OF` lookup before the backfill then finds each row by the values +//! it held at the cutoff. + +use std::collections::BTreeSet; + +use redb::WriteTransaction; + +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::document::read::decode::decode_scanned_document; +use crate::data::executor::sparse_body_format::SparseBodyFormatRef; +use crate::engine::document::predicate::IndexPredicate; +use crate::engine::document::store::{StorageKey, extract_index_values}; +use crate::engine::sparse::btree_versioned::VersionedIndexEntry; + +/// The index a versioned backfill builds. +pub(super) struct VersionedBackfill<'a> { + pub database_id: u64, + pub tid: u64, + pub collection: &'a str, + pub path: &'a str, + pub is_array: bool, + pub case_insensitive: bool, + /// The partial-index predicate. A version it rejects holds no value. + pub predicate: Option<&'a IndexPredicate>, + /// How the collection's stored bodies are encoded. + pub body_format: SparseBodyFormatRef<'a>, +} + +impl CoreLoop { + /// Write the versioned index entries of every version of every row, + /// each at its version's system time, within `txn`. Does NOT commit. + /// + /// A version that does not decode fails the backfill: an index built + /// without it misses the row at every cutoff the version covers. + pub(super) fn backfill_versioned_index( + &self, + txn: &WriteTransaction, + b: &VersionedBackfill<'_>, + ) -> crate::Result<()> { + let mut row: Option = None; + let mut held: BTreeSet = BTreeSet::new(); + self.sparse.versioned_for_each_version( + b.database_id, + b.tid, + b.collection, + |doc_id, sys_from_ms, body| { + if row != Some(*doc_id) { + row = Some(*doc_id); + held.clear(); + } + let values = match body { + Some(body) => version_values(body, b)?, + None => BTreeSet::new(), + }; + for value in held.difference(&values) { + self.sparse.versioned_index_tombstone_in_txn( + txn, + index_entry(b, doc_id, value, sys_from_ms), + )?; + } + for value in &values { + self.sparse.versioned_index_put_in_txn( + txn, + index_entry(b, doc_id, value, sys_from_ms), + )?; + } + held = values; + Ok(()) + }, + ) + } +} + +/// The versioned index entry of `value` for row `doc_id` at `sys_from_ms`. +fn index_entry<'a>( + b: &VersionedBackfill<'a>, + doc_id: &'a StorageKey, + value: &'a str, + sys_from_ms: i64, +) -> VersionedIndexEntry<'a> { + VersionedIndexEntry { + database_id: b.database_id, + tenant: b.tid, + coll: b.collection, + field: b.path, + value, + doc_id, + sys_from_ms, + } +} + +/// The values one live version contributes to the index: none when the +/// partial-index predicate rejects it, lowercased for a case-insensitive +/// index. +fn version_values(body: &[u8], b: &VersionedBackfill<'_>) -> crate::Result> { + let doc = decode_scanned_document(body, b.body_format)?; + if let Some(predicate) = b.predicate + && !predicate.evaluate_json(&doc) + { + return Ok(BTreeSet::new()); + } + Ok(extract_index_values(&doc, b.path, b.is_array) + .into_iter() + .map(|v| { + if b.case_insensitive { + v.to_lowercase() + } else { + v + } + }) + .collect()) +} diff --git a/nodedb/src/data/executor/handlers/document/index_fetch.rs b/nodedb/src/data/executor/handlers/document/index_fetch.rs index 5d1bda4a4..a3652c5bd 100644 --- a/nodedb/src/data/executor/handlers/document/index_fetch.rs +++ b/nodedb/src/data/executor/handlers/document/index_fetch.rs @@ -42,6 +42,9 @@ pub(in crate::data::executor) struct IndexedFetchParams<'a> { pub projection: &'a [String], pub limit: usize, pub offset: usize, + /// `FOR SYSTEM_TIME AS OF ` cutoff on a bitemporal collection, + /// `None` for the current state. + pub system_as_of_ms: Option, } impl CoreLoop { @@ -157,6 +160,7 @@ impl CoreLoop { projection: _projection, limit, offset, + system_as_of_ms, } = params; debug!( core = self.core_id, @@ -170,15 +174,29 @@ impl CoreLoop { let database_id = task.request.database_id.as_u64(); let bitemporal = self.is_bitemporal(database_id, tid, collection); - let doc_engine = - crate::engine::document::store::DocumentEngine::new(&self.sparse, database_id, tid); - let mut doc_ids: Vec = - match doc_engine.index_lookup(collection, path, value, bitemporal) { - Ok(ids) => ids, - Err(e) => { - return self.response_error(task, ErrorCode::from(e)); - } - }; + // An `AS OF` read answers from the versioned index at its cutoff: a + // row matches when its newest entry at or before the cutoff is live. + // The planner refuses `AS OF` on a plain collection, so only a + // bitemporal collection carries a cutoff. + let as_of_ms = system_as_of_ms.filter(|_| bitemporal); + let looked_up = match as_of_ms { + Some(ms) => self.sparse.versioned_index_lookup_as_of( + database_id, + tid, + collection, + path, + value, + Some(ms), + ), + None => { + crate::engine::document::store::DocumentEngine::new(&self.sparse, database_id, tid) + .index_lookup(collection, path, value, bitemporal) + } + }; + let mut doc_ids: Vec = match looked_up { + Ok(ids) => ids, + Err(e) => return self.response_error(task, ErrorCode::from(e)), + }; // Strict collections store Binary Tuple bytes; the response codec // expects msgpack maps. Decode-then-encode here so cross-engine @@ -234,7 +252,9 @@ impl CoreLoop { } } }; - if let Some(txn_id) = task.request.txn_id { + // An `AS OF` read never sees the overlay, whose staged bodies are + // current-version only, as the document scan does. + if let (Some(txn_id), None) = (task.request.txn_id, as_of_ms) { let (is_array, case_insensitive) = self.index_path_flags(&config_key, path); if let Err(e) = self.merge_overlay_into_index_lookup( super::super::transaction::overlay::IndexOverlayMergeParams { @@ -267,20 +287,25 @@ impl CoreLoop { if rows.len() >= limit { break; } - // Bitemporal collections keep the current body on the versioned - // document table; the plain DOCUMENTS table is empty for them. // A doc ID the overlay merge added or superseded above has no // correct body in base storage — `overlay_or_base_body` prefers - // the staged `Put` bytes and only falls back to a base fetch - // when the overlay has nothing staged for this surrogate. - let fetched = self.overlay_or_base_body(task.request.txn_id, &coll_key, doc_id, || { - if bitemporal { - self.sparse - .versioned_get_current(database_id, tid, collection, doc_id) - } else { - self.sparse.get(database_id, tid, collection, doc_id) - } - }); + // the staged `Put` bytes and only falls back to a base fetch of + // the current version when the overlay has nothing staged for + // this surrogate. + // An `AS OF` read takes each row's version as of the cutoff. + let fetched = match as_of_ms { + Some(ms) => self.sparse.versioned_get_as_of( + database_id, + tid, + collection, + doc_id, + Some(ms), + None, + ), + None => self.overlay_or_base_body(task.request.txn_id, &coll_key, doc_id, || { + self.current_row_body(database_id, tid, collection, doc_id) + }), + }; match fetched { Ok(Some(bytes)) => { // A row matching the indexed term but failing the residual diff --git a/nodedb/src/data/executor/handlers/document/index_maintenance.rs b/nodedb/src/data/executor/handlers/document/index_maintenance.rs index 1b1c37db5..b8d2047e4 100644 --- a/nodedb/src/data/executor/handlers/document/index_maintenance.rs +++ b/nodedb/src/data/executor/handlers/document/index_maintenance.rs @@ -81,7 +81,9 @@ impl CoreLoop { // cap matches the Data Plane's other collection-wide scans; rows // beyond this are handled by a future chunked backfill (see // `scan_documents_chunked`). - let docs = match self.sparse.scan_documents( + // Each row at its current version: a bitemporal collection holds its + // rows only as versions. + let docs = match self.current_rows( task.request.database_id.as_u64(), tid, collection, @@ -123,6 +125,9 @@ impl CoreLoop { // Index keys are accumulated here and written in one sorted pass after // the scan; see the bulk insert below. let mut pending_keys: Vec = Vec::with_capacity(docs.len()); + // A bitemporal collection's lookups read the versioned index, which + // the history walk below fills. The current rows still judge UNIQUE. + let bitemporal = self.is_bitemporal(task.request.database_id.as_u64(), tid, collection); for (doc_id, bytes) in &docs { // A row skipped here is a row the finished index permanently omits, @@ -165,6 +170,9 @@ impl CoreLoop { if unique { seen.insert(stored.clone(), *doc_id); } + if bitemporal { + continue; + } pending_keys.push(crate::engine::sparse::btree_index::index_key_for( crate::engine::sparse::btree_index::IndexEntryTxn { database_id: task.request.database_id.as_u64(), @@ -186,6 +194,24 @@ impl CoreLoop { if let Err(e) = self.sparse.index_put_sorted_in_txn(&txn, &mut pending_keys) { return self.response_error(task, crate::bridge::envelope::ErrorCode::from(e)); } + // Every version of every row, each at its own system time. + if bitemporal + && let Err(e) = self.backfill_versioned_index( + &txn, + &super::index_backfill_versions::VersionedBackfill { + database_id: task.request.database_id.as_u64(), + tid, + collection, + path, + is_array, + case_insensitive, + predicate: parsed_predicate.as_ref(), + body_format: body_format.as_format_ref(), + }, + ) + { + return self.response_error(task, crate::bridge::envelope::ErrorCode::from(e)); + } if let Err(e) = txn.commit() { return self.response_error( @@ -201,7 +227,7 @@ impl CoreLoop { /// Drop all secondary index entries for a field across the entire collection. /// - /// Calls `SparseEngine::delete_index_entries_for_field` directly. + /// Removes the field's plain and versioned index entries. /// Returns `{"removed": N}` as the response payload. pub(in crate::data::executor) fn execute_drop_document_index( &mut self, @@ -217,12 +243,34 @@ impl CoreLoop { "drop document index" ); - match self.sparse.delete_index_entries_for_field( - task.request.database_id.as_u64(), - tid, - collection, - field, - ) { + // Both index tables: a bitemporal collection keeps its entries in the + // versioned index, any other in the plain one. The other table holds + // nothing for the field, so its removal finds no entry. + // Both removals share one write transaction, so the field leaves both + // tables together or neither. + let database_id = task.request.database_id.as_u64(); + let removed = self.sparse.begin_write().and_then(|txn| { + let plain = self.sparse.delete_index_entries_for_field_in_txn( + &txn, + database_id, + tid, + collection, + field, + )?; + let versioned = self.sparse.versioned_index_drop_field_in_txn( + &txn, + database_id, + tid, + collection, + field, + )?; + txn.commit().map_err(|e| crate::Error::Storage { + engine: "sparse".into(), + detail: format!("drop index commit: {e}"), + })?; + Ok(plain + versioned) + }); + match removed { Ok(removed) => { match super::super::super::response_codec::encode_count("removed", removed) { Ok(bytes) => self.response_with_payload(task, bytes), diff --git a/nodedb/src/data/executor/handlers/document/mod.rs b/nodedb/src/data/executor/handlers/document/mod.rs index d35a3bd29..b3b06fc34 100644 --- a/nodedb/src/data/executor/handlers/document/mod.rs +++ b/nodedb/src/data/executor/handlers/document/mod.rs @@ -6,6 +6,7 @@ //! helpers). pub mod apply_balance_delta; +pub mod index_backfill_versions; pub mod index_fetch; pub mod index_maintenance; pub mod read; diff --git a/nodedb/src/data/executor/handlers/document/read/fetch.rs b/nodedb/src/data/executor/handlers/document/read/fetch.rs index 919b5afc6..e6f34891c 100644 --- a/nodedb/src/data/executor/handlers/document/read/fetch.rs +++ b/nodedb/src/data/executor/handlers/document/read/fetch.rs @@ -20,15 +20,13 @@ //! `effective_schema` is `None` and the shared sort/window/computed/projection //! pipeline operates on a uniform shape. -use std::cell::Cell; - use nodedb_types::StorageKey; use super::audit_body::{inject_temporal_columns, strict_audit_body}; use super::fetch_types::FetchedRows; use super::{DocFetchParams, DocScanMode}; use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::core_loop::filter_match::matches_with_resolved_schema; +use crate::data::executor::core_loop::filter_match::{HeldEvalError, matches_with_resolved_schema}; use crate::data::executor::scan_normalize::sparse_body_to_msgpack; use crate::data::executor::sparse_body_format::{SparseBodyFormat, SparseBodyFormatRef}; use crate::data::executor::task::ExecutionTask; @@ -69,26 +67,21 @@ impl CoreLoop { // shared sort/window/computed/projection pipeline (which scans // msgpack) operates uniformly, then hand it downstream with no // schema (bodies are already normalized). - // `versioned_scan_as_of` takes an infallible - // `Fn(&StorageKey, &[u8]) -> bool` predicate, so a - // division/modulo-by-zero is captured via this `Cell` - // side-channel and checked once the scan returns, rather - // than silently folded away. - let predicate_err: Cell> = Cell::new(None); - let predicate = - |doc_id: &StorageKey, body: &[u8]| match matches_with_resolved_schema( - strict_schema, - filter_predicates, - doc_id, - body, - &identity_column, - ) { - Ok(b) => b, - Err(e) => { - predicate_err.set(Some(e)); - false - } - }; + // `versioned_scan_as_of` takes an infallible predicate, so a + // division/modulo-by-zero is held and returned once the scan + // ends, rather than silently folded away. + let held = HeldEvalError::default(); + let predicate = |doc_id: &StorageKey, body: &[u8]| { + held.check(|| { + matches_with_resolved_schema( + strict_schema, + filter_predicates, + doc_id, + body, + &identity_column, + ) + }) + }; let raw = self.sparse.versioned_scan_as_of( crate::engine::sparse::btree_versioned::VersionedScanParams { database_id: task.request.database_id.as_u64(), @@ -101,9 +94,7 @@ impl CoreLoop { &predicate, &stop, )?; - if let Some(e) = predicate_err.take() { - return Err(crate::Error::from(e)); - } + let raw = held.finish::<_, crate::Error>(raw)?; let rows = raw .into_iter() .map(|(doc_id, body)| { @@ -131,22 +122,19 @@ impl CoreLoop { // normalized to MessagePack and gets the synthetic `_ts_*` // temporal columns injected BEFORE the shared downstream runs, // so a user can `SELECT` / `ORDER BY` / project on them. - // See the `AsOf` arm above for the `Cell` side-channel rationale. - let predicate_err: Cell> = Cell::new(None); - let predicate = - |doc_id: &StorageKey, body: &[u8]| match matches_with_resolved_schema( - strict_schema, - filter_predicates, - doc_id, - body, - &identity_column, - ) { - Ok(b) => b, - Err(e) => { - predicate_err.set(Some(e)); - false - } - }; + // See the `AsOf` arm above for the held-error rationale. + let held = HeldEvalError::default(); + let predicate = |doc_id: &StorageKey, body: &[u8]| { + held.check(|| { + matches_with_resolved_schema( + strict_schema, + filter_predicates, + doc_id, + body, + &identity_column, + ) + }) + }; let raw = self.sparse.versioned_scan_all( crate::engine::sparse::btree_versioned::VersionedScanParams { database_id: task.request.database_id.as_u64(), @@ -159,9 +147,7 @@ impl CoreLoop { &predicate, &stop, )?; - if let Some(e) = predicate_err.take() { - return Err(crate::Error::from(e)); - } + let raw = held.finish::<_, crate::Error>(raw)?; let mut rows: Vec<(StorageKey, Vec)> = Vec::with_capacity(raw.len()); for row in raw { let msgpack_body = match strict_schema { @@ -232,10 +218,9 @@ impl CoreLoop { ); // `scan_documents_filtered`/`versioned_scan_as_of` take an infallible - // predicate, so a division/modulo-by-zero is captured via this `Cell` - // side-channel and checked once every branch below returns, rather - // than silently folded away. - let predicate_err: Cell> = Cell::new(None); + // predicate, so a division/modulo-by-zero is held and returned once + // every branch below returns, rather than silently folded away. + let held = HeldEvalError::default(); let matches = |key: &StorageKey, value: &[u8]| -> bool { if filter_predicates.is_empty() { return true; @@ -251,19 +236,15 @@ impl CoreLoop { } else { value }; - match matches_with_resolved_schema( - strict_schema, - filter_predicates, - key, - value, - &identity_column, - ) { - Ok(b) => b, - Err(e) => { - predicate_err.set(Some(e)); - false - } - } + held.check(|| { + matches_with_resolved_schema( + strict_schema, + filter_predicates, + key, + value, + &identity_column, + ) + }) }; let rows: Vec<(StorageKey, Vec)> = if filter_predicates.is_empty() { @@ -319,9 +300,7 @@ impl CoreLoop { )? }; - if let Some(e) = predicate_err.take() { - return Err(crate::Error::from(e)); - } + let rows = held.finish::<_, crate::Error>(rows)?; // A vector-primary collection's sparse rows are `zerompk` TAGGED // metadata sidecars, not document bodies. Normalize them here, at the diff --git a/nodedb/src/data/executor/handlers/document/read/scan.rs b/nodedb/src/data/executor/handlers/document/read/scan.rs index 8f0d101ee..2a193a6e3 100644 --- a/nodedb/src/data/executor/handlers/document/read/scan.rs +++ b/nodedb/src/data/executor/handlers/document/read/scan.rs @@ -204,29 +204,24 @@ impl CoreLoop { crate::types::TenantId::new(tid), collection.to_string(), ); - // `merge_overlay_into_scan` takes an infallible - // `Fn(&StorageKey, &[u8]) -> bool` predicate, so a - // division/modulo-by-zero is captured via this `Cell` - // side-channel and checked once the merge returns. - let predicate_err: std::cell::Cell> = - std::cell::Cell::new(None); + // `merge_overlay_into_scan` takes an infallible predicate, + // so a division/modulo-by-zero is held and returned once + // the merge ends. + let held = + crate::data::executor::core_loop::filter_match::HeldEvalError::default(); let matches = |row_key: &StorageKey, value: &[u8]| -> bool { if filter_predicates.is_empty() { return true; } - match crate::data::executor::core_loop::filter_match::matches_with_resolved_schema( - effective_schema.as_ref(), - &filter_predicates, - row_key, - value, - &identity_column, - ) { - Ok(b) => b, - Err(e) => { - predicate_err.set(Some(e)); - false - } - } + held.check(|| { + crate::data::executor::core_loop::filter_match::matches_with_resolved_schema( + effective_schema.as_ref(), + &filter_predicates, + row_key, + value, + &identity_column, + ) + }) }; // A vector-primary row stages its vector with its sidecar; // its merge renders the sidecar the way the fetch stage @@ -252,8 +247,8 @@ impl CoreLoop { } else { self.merge_overlay_into_scan(txn_id, &coll_key, &mut filtered, &matches); } - if let Some(e) = predicate_err.take() { - return self.response_error(task, ErrorCode::from(e)); + if let Err(e) = held.finish::<_, ErrorCode>(()) { + return self.response_error(task, e); } } diff --git a/nodedb/src/data/executor/handlers/document/resolve/apply.rs b/nodedb/src/data/executor/handlers/document/resolve/apply.rs index 302842758..0fea3a459 100644 --- a/nodedb/src/data/executor/handlers/document/resolve/apply.rs +++ b/nodedb/src/data/executor/handlers/document/resolve/apply.rs @@ -171,8 +171,9 @@ impl CoreLoop { let database_id = task.request.database_id.as_u64(); for mutation in mutations { let row_key = StorageKey::for_surrogate(mutation.surrogate()); - let current = - self.doc_current_bytes(database_id, tid, mutation.collection().as_str(), &row_key)?; + let current = self + .current_row_body(database_id, tid, mutation.collection().as_str(), &row_key) + .map_err(ErrorCode::from)?; if current.as_deref() != mutation.precondition() { return Err(ErrorCode::OllpRetryRequired); } diff --git a/nodedb/src/data/executor/handlers/document/resolve/bulk.rs b/nodedb/src/data/executor/handlers/document/resolve/bulk.rs index 3eb5b2a07..617f79262 100644 --- a/nodedb/src/data/executor/handlers/document/resolve/bulk.rs +++ b/nodedb/src/data/executor/handlers/document/resolve/bulk.rs @@ -21,9 +21,9 @@ use super::context::{ resolved_response_payload, }; use crate::bridge::envelope::ErrorCode; -use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::doc_format; +use crate::data::executor::handlers::bulk_dml::decode_bulk_filters; use crate::data::executor::handlers::bulk_dml::update_project::{ MatchedRows, ProjectUpdateRows, ProjectedUpdateRow, }; @@ -88,7 +88,7 @@ impl CoreLoop { )?; } - let filters = decode_filters(filter_bytes)?; + let filters = decode_bulk_filters(filter_bytes).map_err(ErrorCode::from)?; let matched = self.resolve_matched_rows(task, &ctx, collection, &filters)?; // The same projection the live handler runs before it writes anything, @@ -102,6 +102,10 @@ impl CoreLoop { updates, strict_schema: ctx.strict_schema.as_ref(), declared_primary_key, + // Stamped as the live handler stamps it, so a strict + // bitemporal image takes the bitemporal tuple layout. The + // apply mints its own system time on write. + bitemporal_sys_from_ms: ctx.bitemporal.then(|| self.bitemporal_now_ms()), }) .map_err(ErrorCode::from)?; @@ -165,7 +169,7 @@ impl CoreLoop { resolved_sum_targets, } = args; let ctx = self.doc_resolve_ctx(task, tid, collection); - let filters = decode_filters(filter_bytes)?; + let filters = decode_bulk_filters(filter_bytes).map_err(ErrorCode::from)?; let matched = self.resolve_matched_rows(task, &ctx, collection, &filters)?; let mut mutations = Vec::with_capacity(matched.len()); @@ -206,13 +210,3 @@ impl CoreLoop { }) } } - -/// Decode a plan's filter payload. Empty means "no WHERE clause" — every row. -fn decode_filters(filter_bytes: &[u8]) -> Result, ErrorCode> { - if filter_bytes.is_empty() { - return Ok(Vec::new()); - } - zerompk::from_msgpack(filter_bytes).map_err(|e| ErrorCode::Internal { - detail: format!("deserialize filters: {e}"), - }) -} diff --git a/nodedb/src/data/executor/handlers/document/resolve/context.rs b/nodedb/src/data/executor/handlers/document/resolve/context.rs index e07b9aa26..39e5a9ece 100644 --- a/nodedb/src/data/executor/handlers/document/resolve/context.rs +++ b/nodedb/src/data/executor/handlers/document/resolve/context.rs @@ -64,34 +64,16 @@ impl CoreLoop { } } - /// The row's raw stored body, or `None` when absent. Reads through the - /// same current-state view the live write path uses; becomes the - /// mutation's `precondition`. + /// The row's current stored body as a resolve error type, or `None` when + /// absent. It becomes the mutation's `precondition`. pub(super) fn doc_resolve_read( &self, ctx: &DocResolveCtx, collection: &str, row_key: &StorageKey, ) -> Result>, ErrorCode> { - self.doc_current_bytes(ctx.database_id, ctx.tid, collection, row_key) - } - - /// The same current-state read, for a caller with no [`DocResolveCtx`] — - /// the apply path, which checks one precondition per mutation. - pub(in crate::data::executor) fn doc_current_bytes( - &self, - database_id: u64, - tid: u64, - collection: &str, - row_key: &StorageKey, - ) -> Result>, ErrorCode> { - let read = if self.is_bitemporal(database_id, tid, collection) { - self.sparse - .versioned_get_current(database_id, tid, collection, row_key) - } else { - self.sparse.get(database_id, tid, collection, row_key) - }; - read.map_err(ErrorCode::from) + self.current_row_body(ctx.database_id, ctx.tid, collection, row_key) + .map_err(ErrorCode::from) } } diff --git a/nodedb/src/data/executor/handlers/document/resolve/txn_rows.rs b/nodedb/src/data/executor/handlers/document/resolve/txn_rows.rs index 04e0aac6a..35a2eb763 100644 --- a/nodedb/src/data/executor/handlers/document/resolve/txn_rows.rs +++ b/nodedb/src/data/executor/handlers/document/resolve/txn_rows.rs @@ -3,8 +3,6 @@ //! The rows a predicate document write matches: the base rows its filters //! match, and, inside a transaction, that transaction's own staged writes. -use std::cell::Cell; - use crate::bridge::envelope::ErrorCode; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; @@ -15,7 +13,8 @@ use crate::types::TenantId; use super::context::DocResolveCtx; impl CoreLoop { - /// Every `(row key, current body)` that `filters` match in `collection`. + /// Every `(row key, current body)` that `filters` match in `collection`, + /// each at the current version the mutation's precondition reads. /// /// A request that carries a transaction id reads its own writes: a /// staged delete drops its row, a staged put is matched again against @@ -27,39 +26,17 @@ impl CoreLoop { collection: &str, filters: &[ScanFilter], ) -> Result)>, ErrorCode> { - let keys = self - .scan_matching_documents(ctx.database_id, ctx.tid, collection, filters) - .map_err(ErrorCode::from)?; - let mut rows = Vec::with_capacity(keys.len()); - for key in keys { - if let Some(stored) = self.doc_resolve_read(ctx, collection, &key)? { - rows.push((key, stored)); - } - } - let Some(txn_id) = task.request.txn_id else { - return Ok(rows); - }; - - let coll_key = ( - task.request.database_id, - TenantId::new(ctx.tid), - collection.to_string(), - ); - let raw_matches = self.strict_aware_matcher(ctx.database_id, ctx.tid, collection, filters); - // The merge takes an infallible predicate, so an evaluation error - // (division by zero) is held here and raised once the merge returns. - let predicate_err: Cell> = Cell::new(None); - let matches = |row_key: &StorageKey, body: &[u8]| match raw_matches(row_key, body) { - Ok(matched) => matched, - Err(e) => { - predicate_err.set(Some(e)); - false + let rows = match task.request.txn_id { + Some(txn_id) => { + let coll_key = ( + task.request.database_id, + TenantId::new(ctx.tid), + collection.to_string(), + ); + self.txn_matching_rows(txn_id, &coll_key, filters) } + None => self.current_matching_rows(ctx.database_id, ctx.tid, collection, filters), }; - self.merge_overlay_into_scan(txn_id, &coll_key, &mut rows, &matches); - if let Some(e) = predicate_err.take() { - return Err(ErrorCode::from(crate::Error::from(e))); - } - Ok(rows) + rows.map_err(ErrorCode::from) } } diff --git a/nodedb/src/data/executor/handlers/facet.rs b/nodedb/src/data/executor/handlers/facet.rs index 5c104bc24..bfc53ed21 100644 --- a/nodedb/src/data/executor/handlers/facet.rs +++ b/nodedb/src/data/executor/handlers/facet.rs @@ -50,18 +50,20 @@ impl CoreLoop { } }; - let matching_ids = match self.scan_matching_documents( + // Each matching row at its current version: the newest version on a + // bitemporal collection, the plain table on any other. + let matching_rows = match self.current_matching_rows( task.request.database_id.as_u64(), tid, collection, &filters, ) { - Ok(ids) => ids, + Ok(rows) => rows, Err(e) => return self.response_error(task, e), }; let matching_set: HashSet = - matching_ids.iter().copied().collect(); + matching_rows.iter().map(|(key, _)| *key).collect(); // Step 2: For each facet field, count values. let mut facet_result = serde_json::Map::new(); @@ -78,7 +80,7 @@ impl CoreLoop { collection, field, &matching_set, - &matching_ids, + &matching_rows, ) { Ok(counts) => counts, Err(e) => return self.response_error(task, e), @@ -135,7 +137,7 @@ impl CoreLoop { collection: &str, field: &str, matching_set: &HashSet, - matching_ids: &[nodedb_types::StorageKey], + matching_rows: &[(nodedb_types::StorageKey, Vec)], ) -> crate::Result> { // Fast path: index-backed counting with filtered doc set. let groups = self.sparse.scan_index_groups_filtered( @@ -156,44 +158,39 @@ impl CoreLoop { // strict collection's Binary Tuple untouched — `extract_field` then // finds nothing in every row and the facet reports an empty count // rather than an error, so the facet silently disagrees with the scan - // that produced `matching_ids`. + // that produced `matching_rows`. let body_format = self.sparse_body_format( crate::types::DatabaseId::new(database_id), crate::types::TenantId::new(tid), collection, ); let mut counts: HashMap = HashMap::new(); - for key in matching_ids { - if let Some(bytes) = self.sparse.get(database_id, tid, collection, key)? { - let mp = crate::data::executor::scan_normalize::sparse_body_to_msgpack( - &bytes, - body_format.as_format_ref(), - ); - if let Some((start, end)) = nodedb_query::msgpack_scan::extract_field(&mp, 0, field) - { - let value_str = if let Some(s) = - nodedb_query::msgpack_scan::read_str(&mp, start) - { - s.to_string() - } else if let Some(i) = nodedb_query::msgpack_scan::read_integer(&mp, start) { - i.to_string() - } else if let Some(f) = nodedb_query::msgpack_scan::read_f64(&mp, start) { - f.to_string() - } else if let Some(b) = nodedb_query::msgpack_scan::read_bool(&mp, start) { - b.to_string() - } else if nodedb_query::msgpack_scan::read_null(&mp, start) { - continue; - } else { - // Complex value — stringify via transcoder. - nodedb_types::msgpack_to_json_string(&mp[start..end]).map_err(|e| { - crate::Error::Serialization { - format: "msgpack".to_string(), - detail: format!("facet value of '{field}' in {key}: {e}"), - } - })? - }; - *counts.entry(value_str).or_default() += 1; - } + for (key, bytes) in matching_rows { + let mp = crate::data::executor::scan_normalize::sparse_body_to_msgpack( + bytes, + body_format.as_format_ref(), + ); + if let Some((start, end)) = nodedb_query::msgpack_scan::extract_field(&mp, 0, field) { + let value_str = if let Some(s) = nodedb_query::msgpack_scan::read_str(&mp, start) { + s.to_string() + } else if let Some(i) = nodedb_query::msgpack_scan::read_integer(&mp, start) { + i.to_string() + } else if let Some(f) = nodedb_query::msgpack_scan::read_f64(&mp, start) { + f.to_string() + } else if let Some(b) = nodedb_query::msgpack_scan::read_bool(&mp, start) { + b.to_string() + } else if nodedb_query::msgpack_scan::read_null(&mp, start) { + continue; + } else { + // Complex value — stringify via transcoder. + nodedb_types::msgpack_to_json_string(&mp[start..end]).map_err(|e| { + crate::Error::Serialization { + format: "msgpack".to_string(), + detail: format!("facet value of '{field}' in {key}: {e}"), + } + })? + }; + *counts.entry(value_str).or_default() += 1; } } diff --git a/nodedb/src/data/executor/handlers/graph_rag.rs b/nodedb/src/data/executor/handlers/graph_rag.rs index 78c8d5231..3763f31b2 100644 --- a/nodedb/src/data/executor/handlers/graph_rag.rs +++ b/nodedb/src/data/executor/handlers/graph_rag.rs @@ -16,11 +16,12 @@ use std::collections::HashMap; -use nodedb_types::{RowIdentity, Surrogate}; +use nodedb_types::Surrogate; use nodedb_vector::SearchResult; use tracing::{debug, warn}; use super::graph_expansion::{GraphExpansionParams, GraphSeeds}; +use super::rag_key::RagKey; use crate::bridge::envelope::{ErrorCode, Response}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::response_codec::{ @@ -33,21 +34,20 @@ use crate::query::fusion::{FusedResult, RankedResult, reciprocal_rank_fusion_wei /// Result of a successful vector search. /// /// - `Vec` is the raw HNSW output, for reporting candidate counts. -/// - `HashMap` maps each hit's *reporting key* to `(rank, distance)`. That key -/// is what RRF fuses on and what the response returns: the client-visible -/// identity of the row. +/// - `HashMap` maps each hit's [`RagKey`] to `(rank, distance)`. That key is +/// what RRF fuses on. The response renders it after fusion. /// - `Vec` is the same hits in the identity currency, ready to seed /// graph expansion with no translation. type VectorNodeScores = ( Vec, - HashMap, + HashMap, Vec, ); /// Parameters for `build_rag_response` and [`rag_response_body`]. pub(crate) struct RagResponseParams<'a> { - pub fused: &'a [FusedResult], - pub vector_scores: &'a HashMap, + pub fused: &'a [FusedResult], + pub vector_scores: &'a HashMap, pub hop_distances: &'a HashMap, pub vector_candidate_count: usize, pub graph_expanded_count: usize, @@ -189,15 +189,11 @@ impl CoreLoop { // graph leg seeds from directly — no name is minted to seed a walk that // would only hash it straight back to the same node. // - // The reporting key is resolved once per hit: the graph node name when - // the surrogate is bound to one, otherwise the surrogate's - // client-visible decimal identity. Falling back to the identity rather - // than to an index-local sentinel is what lets a hit fuse with the - // *text* leg, which keys on exactly that; the old `__local_{hnsw_id}` - // sentinel could match nothing and leaked an internal index id into - // the response's `node_id`. + // The fusion key is resolved once per hit: the graph node name when + // the surrogate is bound to one, otherwise the surrogate itself. The + // text leg keys by the same rule, so the two legs meet on one key. let csr = self.csr_partition(database_id, tenant_id); - let mut vector_scores: HashMap = HashMap::new(); + let mut vector_scores: HashMap = HashMap::new(); let mut seeds: Vec = Vec::with_capacity(hits.len()); let mut vector_results: Vec = Vec::with_capacity(hits.len()); for (rank, (result, surrogate)) in hits.into_iter().enumerate() { @@ -205,15 +201,10 @@ impl CoreLoop { seeds.push(s); } let key = match surrogate { - Some(s) => csr - .and_then(|c| c.node_id_for_surrogate(s)) - .map(RowIdentity::from_user_key) - .unwrap_or_else(|| RowIdentity::for_surrogate(s)), - // No surrogate at all: the vector entry predates surrogate - // plumbing, so it has no cross-engine identity. It still ranks - // in the vector leg under a key that deliberately matches - // nothing else. - None => RowIdentity::from_user_key(format!("__unbound_{}", result.id)), + Some(s) => RagKey::of_hit(csr.and_then(|c| c.node_id_for_surrogate(s)), s), + // An entry with no surrogate has no cross-engine identity. It + // ranks in the vector leg under a key that matches nothing else. + None => RagKey::Unbound(result.id), }; vector_scores.insert(key, (rank, result.distance)); vector_results.push(result); @@ -299,10 +290,16 @@ pub(crate) fn rag_response_body(p: &RagResponseParams<'_>, watermark_lsn: u64) - .get(&f.document_id) .map(|(rank, dist)| (Some(*rank), Some(*dist))) .unwrap_or((None, None)); - let hop_distance = p.hop_distances.get(f.document_id.as_str()).copied(); + let (node_id, surrogate, hop_distance) = match &f.document_id { + RagKey::Node(name) => (name.clone(), None, p.hop_distances.get(name).copied()), + // The response boundary names this row from the catalog. The + // decimal stands only for a row the catalog binds no key to. + RagKey::Row(s) => (s.as_u32().to_string(), Some(s.as_u32()), None), + RagKey::Unbound(_) => (f.document_id.rendered().into_owned(), None, None), + }; GraphRagResult { - // The identity is rendered here, at the response envelope. - node_id: f.document_id.clone().into_string(), + node_id, + surrogate, rrf_score: f.rrf_score, vector_rank, vector_distance, @@ -322,10 +319,10 @@ pub(crate) fn rag_response_body(p: &RagResponseParams<'_>, watermark_lsn: u64) - } } -/// The vector leg as a ranked list, keyed by each hit's reporting key. +/// The vector leg as a ranked list, keyed by each hit's fusion key. pub(crate) fn vector_ranked_list( - vector_scores: &HashMap, -) -> Vec> { + vector_scores: &HashMap, +) -> Vec> { vector_scores .iter() .map(|(node_id, (rank, dist))| RankedResult { @@ -339,15 +336,15 @@ pub(crate) fn vector_ranked_list( /// Sort expanded graph nodes by hop distance and convert to `RankedResult` list. /// -/// A graph node name is the row's client identity, so the list keys on -/// `RowIdentity`. Used by 2-source GraphRAG, 3-source GraphRAG triple, and the -/// cluster coordinator. +/// A graph node name is the row's client key, so the list keys on +/// [`RagKey::Node`]. Used by 2-source GraphRAG, 3-source GraphRAG triple, and +/// the cluster coordinator. pub(crate) fn graph_nodes_to_ranked_results( expanded_nodes: Vec, hop_distances: &HashMap, -) -> Vec> { - // Takes `expanded_nodes` by value so the name moves straight into - // `RowIdentity` below instead of being copied from a borrow. +) -> Vec> { + // Takes `expanded_nodes` by value so the name moves straight into the + // key below instead of being copied from a borrow. let mut sorted: Vec<(String, usize)> = expanded_nodes .into_iter() .map(|node| { @@ -361,7 +358,7 @@ pub(crate) fn graph_nodes_to_ranked_results( .into_iter() .enumerate() .map(|(rank, (node_id, hop_dist))| RankedResult { - document_id: RowIdentity::from_user_key(node_id), + document_id: RagKey::Node(node_id), rank, score: hop_dist as f32, source: "graph", diff --git a/nodedb/src/data/executor/handlers/graph_rag_triple.rs b/nodedb/src/data/executor/handlers/graph_rag_triple.rs index 48d4d2de1..4e30430c2 100644 --- a/nodedb/src/data/executor/handlers/graph_rag_triple.rs +++ b/nodedb/src/data/executor/handlers/graph_rag_triple.rs @@ -12,7 +12,7 @@ use nodedb_fts::FtsSearchParams; use nodedb_fts::posting::QueryMode; -use nodedb_types::{RowIdentity, Surrogate}; +use nodedb_types::Surrogate; use tracing::debug; use crate::bridge::envelope::Response; @@ -21,6 +21,7 @@ use crate::data::executor::handlers::graph_expansion::{GraphExpansionParams, Gra use crate::data::executor::handlers::graph_rag::{ RagResponseParams, graph_nodes_to_ranked_results, vector_ranked_list, }; +use crate::data::executor::handlers::rag_key::RagKey; use crate::data::executor::task::ExecutionTask; use crate::engine::graph::edge_store::Direction; use crate::query::fusion::{RankedResult, reciprocal_rank_fusion_weighted}; @@ -128,7 +129,10 @@ impl CoreLoop { let (vector_k, text_k, graph_k) = rrf_k; let vector_list = vector_ranked_list(&vector_scores); - let text_list = text_ranked_list(&text_results); + let csr = self.csr_partition(task.request.database_id.as_u64(), tenant_id); + let text_list = text_ranked_list(&text_results, |s| { + csr.and_then(|c| c.node_id_for_surrogate(s)) + }); let graph_expanded_count = expanded_nodes.len(); let graph_list = graph_nodes_to_ranked_results(expanded_nodes, &hop_distances); @@ -192,13 +196,16 @@ impl CoreLoop { } } -/// The BM25 leg as a ranked list. A text hit keys on its row's surrogate -/// identity, the key a vector hit falls back to. -pub(crate) fn text_ranked_list(hits: &[(Surrogate, f32)]) -> Vec> { +/// The BM25 leg as a ranked list. A text hit keys as a vector hit does: the +/// graph node `name_of` names for its surrogate, else the surrogate. +pub(crate) fn text_ranked_list<'n>( + hits: &[(Surrogate, f32)], + name_of: impl Fn(Surrogate) -> Option<&'n str>, +) -> Vec> { hits.iter() .enumerate() .map(|(rank, (doc_id, score))| RankedResult { - document_id: RowIdentity::for_surrogate(*doc_id), + document_id: RagKey::of_hit(name_of(*doc_id), *doc_id), rank, score: *score, source: "text", diff --git a/nodedb/src/data/executor/handlers/mod.rs b/nodedb/src/data/executor/handlers/mod.rs index 949c80a27..da821b323 100644 --- a/nodedb/src/data/executor/handlers/mod.rs +++ b/nodedb/src/data/executor/handlers/mod.rs @@ -16,6 +16,7 @@ pub mod columnar_write; pub mod compact; pub mod control; pub mod convert; +pub mod convert_bitemporal; pub mod document; pub mod facet; pub mod fts_analyzer; @@ -52,6 +53,7 @@ pub(super) mod provider_scan; pub(super) mod provider_scan_compute; pub mod purge; pub mod query_collection_size; +pub mod rag_key; pub mod reclaim; pub(super) mod reclaim_retry; pub mod recursive; @@ -84,6 +86,7 @@ mod timeseries_wal_payload; pub mod transaction; pub mod truncate; pub mod truncate_response; +pub mod truncate_row; pub mod unbound_surrogate; pub mod unregister_collection; pub mod unregister_materialized_view; diff --git a/nodedb/src/data/executor/handlers/point/apply_delete.rs b/nodedb/src/data/executor/handlers/point/apply_delete.rs index fd9860c42..778d984b9 100644 --- a/nodedb/src/data/executor/handlers/point/apply_delete.rs +++ b/nodedb/src/data/executor/handlers/point/apply_delete.rs @@ -12,7 +12,7 @@ use redb::WriteTransaction; use tracing::warn; use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::enforcement::{append_only, period_lock, retention}; +use crate::data::executor::enforcement::{append_only, retention}; use nodedb_physical::physical_plan::ResolvedSumTarget; use nodedb_types::Surrogate; @@ -166,7 +166,7 @@ impl CoreLoop { if let Some(ref body) = prior { if enforce && let Some(config) = self.doc_configs.get(&config_key) { run_delete_enforcement( - &self.sparse, + self, database_id, tid, collection, @@ -177,54 +177,20 @@ impl CoreLoop { } let sys_from = carried_sys_from.unwrap_or_else(|| self.bitemporal_now_ms()); bitemporal_sys_from_ms = Some(sys_from); - self.sparse.versioned_tombstone_in_txn( + // The tombstone version, plus an index tombstone for every + // current value, so a lookup at or after `sys_from` skips + // this row. + bitemporal_index_tuples = self.tombstone_version_in_txn( txn, - database_id, - tid, - collection, - &storage_key, - sys_from, + super::delete_version::VersionTombstone { + database_id, + tid, + collection, + storage_key: &storage_key, + current_body: body, + sys_from_ms: sys_from, + }, )?; - // Index tombstones: reflect every current value so - // `index_lookup_as_of` at or after `sys_from` skips this - // doc_id. `body` is the STORED bytes (MessagePack for - // schemaless, Binary Tuple for strict) — use the - // storage-mode-aware decoder so strict bitemporal deletes - // also tombstone their secondary-index entries instead of - // silently skipping this loop. - if let Some(config) = self.doc_configs.get(&config_key) { - // A body that will not decode leaves every index entry it - // owns un-tombstoned, so the deleted row stays findable by - // its old indexed values. That must fail the delete, not - // skip the loop. - let doc = self.decode_stored_document(config, body)?; - for path in config.index_paths.clone() { - for v in crate::engine::document::store::extract_index_values( - &doc, - &path.path, - path.is_array, - ) { - let value = if path.case_insensitive { - v.to_lowercase() - } else { - v - }; - self.sparse.versioned_index_tombstone_in_txn( - txn, - crate::engine::sparse::btree_versioned::VersionedIndexEntry { - database_id, - tenant: tid, - coll: collection, - field: &path.path, - value: &value, - doc_id: &storage_key, - sys_from_ms: sys_from, - }, - )?; - bitemporal_index_tuples.push((path.path.clone(), value)); - } - } - } } prior } else { @@ -233,7 +199,7 @@ impl CoreLoop { .sparse .get(database_id, tid, collection, &storage_key)?; run_delete_enforcement( - &self.sparse, + self, database_id, tid, collection, @@ -422,7 +388,7 @@ impl CoreLoop { /// These checks have no persistent side effect, so a violation here /// simply aborts before the write. pub(in crate::data::executor) fn run_delete_enforcement( - sparse: &crate::engine::sparse::btree::SparseEngine, + core: &CoreLoop, database_id: u64, tid: u64, collection: &str, @@ -435,8 +401,7 @@ pub(in crate::data::executor) fn run_delete_enforcement( if let Some(ref pl) = config.enforcement.period_lock && let Some(old_bytes) = old_value { - period_lock::check_period_lock( - sparse, + core.check_period_lock_at( database_id, tid, collection, diff --git a/nodedb/src/data/executor/handlers/point/apply_put/enforce.rs b/nodedb/src/data/executor/handlers/point/apply_put/enforce.rs index acce87b39..a66f862d4 100644 --- a/nodedb/src/data/executor/handlers/point/apply_put/enforce.rs +++ b/nodedb/src/data/executor/handlers/point/apply_put/enforce.rs @@ -13,10 +13,10 @@ use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::doc_format; +use crate::data::executor::enforcement::append_only; use crate::data::executor::enforcement::update_transitions::{ UpdateTransitionRules, check_update_transitions, }; -use crate::data::executor::enforcement::{append_only, period_lock}; use crate::types::{DatabaseId, TenantId}; use nodedb_physical::physical_plan::ResolvedSumTarget; @@ -77,16 +77,8 @@ impl CoreLoop { append_only::check_point_put(collection, &config.enforcement, old_value) .map_err(map_enforcement_error)?; if let Some(ref pl) = config.enforcement.period_lock { - period_lock::check_period_lock( - &self.sparse, - database_id, - tid, - collection, - value, - pl, - resolved_targets, - ) - .map_err(map_enforcement_error)?; + self.check_period_lock_at(database_id, tid, collection, value, pl, resolved_targets) + .map_err(map_enforcement_error)?; } // Both images must be readable whenever a transition rule is // configured: skipping a configured check because an image would diff --git a/nodedb/src/data/executor/handlers/point/delete.rs b/nodedb/src/data/executor/handlers/point/delete.rs index 1cc5cfc7c..934b97430 100644 --- a/nodedb/src/data/executor/handlers/point/delete.rs +++ b/nodedb/src/data/executor/handlers/point/delete.rs @@ -338,14 +338,7 @@ impl CoreLoop { } let database_id = task.request.database_id.as_u64(); let storage_key = crate::engine::document::store::StorageKey::for_surrogate(surrogate); - let stored = if self.is_bitemporal(database_id, tid, collection) { - self.sparse - .versioned_get_current(database_id, tid, collection, &storage_key)? - } else { - self.sparse - .get(database_id, tid, collection, &storage_key)? - }; - let Some(body) = stored else { + let Some(body) = self.current_row_body(database_id, tid, collection, &storage_key)? else { return Ok(()); }; let strict_schema = self diff --git a/nodedb/src/data/executor/handlers/point/delete_version.rs b/nodedb/src/data/executor/handlers/point/delete_version.rs new file mode 100644 index 000000000..ddde90941 --- /dev/null +++ b/nodedb/src/data/executor/handlers/point/delete_version.rs @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Removing one row's current version inside a caller-owned transaction. +//! +//! A bitemporal row is never erased. Its delete appends a tombstone version +//! at a system time and tombstones every value the closed version held in the +//! versioned index. History before that system time stays readable with +//! `AS OF SYSTEM TIME`. Any other row is removed from the plain table. +//! +//! The point delete and the bulk delete both land each row here, so the two +//! write the same versions and index tombstones for the same row. + +use redb::WriteTransaction; + +use crate::data::executor::core_loop::CoreLoop; +use crate::engine::document::store::{StorageKey, extract_index_values}; +use crate::engine::sparse::btree_versioned::VersionedIndexEntry; +use crate::types::{DatabaseId, TenantId}; + +/// Inputs to [`CoreLoop::tombstone_version_in_txn`]. +pub(in crate::data::executor) struct VersionTombstone<'a> { + pub database_id: u64, + pub tid: u64, + pub collection: &'a str, + pub storage_key: &'a StorageKey, + /// The stored body of the version the tombstone closes. Its indexed + /// values receive the index tombstones. + pub current_body: &'a [u8], + /// The system time of the tombstone version. + pub sys_from_ms: i64, +} + +/// Inputs to [`CoreLoop::remove_current_row_in_txn`]. +pub(in crate::data::executor) struct CurrentRowRemoval<'a> { + pub database_id: u64, + pub tid: u64, + pub collection: &'a str, + pub storage_key: &'a StorageKey, + /// The tombstone's system time on a bitemporal collection, `None` on any + /// other. + pub bitemporal_sys_from_ms: Option, +} + +/// What [`CoreLoop::remove_current_row_in_txn`] removed. +pub(in crate::data::executor) struct RemovedRow { + /// The removed row's stored body. `None` when no row was there. + pub prior: Option>, + /// The tombstone's system time. `Some` only when a bitemporal row was + /// closed. + pub tombstone_sys_from_ms: Option, +} + +impl CoreLoop { + /// Close a bitemporal row's current version within `txn`. Writes the + /// tombstone version at `sys_from_ms` and an index tombstone for each + /// value the closed version held. Does NOT commit. + /// + /// A registered collection's body must decode. An undecodable body fails + /// the delete, because its index entries left live keep the row findable + /// by its old values. + /// + /// Returns the `(field, value)` tuples tombstoned. On `Err` the caller + /// MUST drop `txn` without committing. + pub(in crate::data::executor) fn tombstone_version_in_txn( + &self, + txn: &WriteTransaction, + p: VersionTombstone<'_>, + ) -> crate::Result> { + let mut tombstoned: Vec<(String, String)> = Vec::new(); + self.write_version_tombstone(txn, p, |field, value| { + tombstoned.push((field.to_string(), value.to_string())); + })?; + Ok(tombstoned) + } + + /// [`Self::tombstone_version_in_txn`] for a caller that records the + /// indexed values from the row's pre-image itself. + pub(in crate::data::executor) fn close_version_in_txn( + &self, + txn: &WriteTransaction, + p: VersionTombstone<'_>, + ) -> crate::Result<()> { + self.write_version_tombstone(txn, p, |_, _| {}) + } + + /// Write the tombstone version and the index tombstones, handing each + /// tombstoned `(field, value)` to `tombstoned`. + fn write_version_tombstone( + &self, + txn: &WriteTransaction, + p: VersionTombstone<'_>, + mut tombstoned: impl FnMut(&str, &str), + ) -> crate::Result<()> { + self.sparse.versioned_tombstone_in_txn( + txn, + p.database_id, + p.tid, + p.collection, + p.storage_key, + p.sys_from_ms, + )?; + let config_key = ( + DatabaseId::new(p.database_id), + TenantId::new(p.tid), + p.collection.to_string(), + ); + let Some(config) = self.doc_configs.get(&config_key) else { + return Ok(()); + }; + let doc = self.decode_stored_document(config, p.current_body)?; + for path in &config.index_paths { + for v in extract_index_values(&doc, &path.path, path.is_array) { + let value = if path.case_insensitive { + v.to_lowercase() + } else { + v + }; + self.sparse.versioned_index_tombstone_in_txn( + txn, + VersionedIndexEntry { + database_id: p.database_id, + tenant: p.tid, + coll: p.collection, + field: &path.path, + value: &value, + doc_id: p.storage_key, + sys_from_ms: p.sys_from_ms, + }, + )?; + tombstoned(&path.path, &value); + } + } + Ok(()) + } + + /// Remove one row's current version within `txn`. Does NOT commit. + /// + /// With a system time, the row's current version is read and closed by + /// [`Self::close_version_in_txn`]. Without one, the row is deleted + /// from the plain table. A row that is absent removes nothing. + /// + /// On `Err` the caller MUST drop `txn` without committing. + pub(in crate::data::executor) fn remove_current_row_in_txn( + &self, + txn: &WriteTransaction, + r: CurrentRowRemoval<'_>, + ) -> crate::Result { + let Some(sys_from_ms) = r.bitemporal_sys_from_ms else { + let prior = self.sparse.delete_in_txn( + txn, + r.database_id, + r.tid, + r.collection, + r.storage_key, + )?; + return Ok(RemovedRow { + prior, + tombstone_sys_from_ms: None, + }); + }; + let Some(prior) = + self.sparse + .versioned_get_current(r.database_id, r.tid, r.collection, r.storage_key)? + else { + return Ok(RemovedRow { + prior: None, + tombstone_sys_from_ms: None, + }); + }; + // The caller records the indexed values from the row's pre-image. + self.close_version_in_txn( + txn, + VersionTombstone { + database_id: r.database_id, + tid: r.tid, + collection: r.collection, + storage_key: r.storage_key, + current_body: &prior, + sys_from_ms, + }, + )?; + Ok(RemovedRow { + prior: Some(prior), + tombstone_sys_from_ms: Some(sys_from_ms), + }) + } +} diff --git a/nodedb/src/data/executor/handlers/point/get.rs b/nodedb/src/data/executor/handlers/point/get.rs index b1087508d..a060f95c4 100644 --- a/nodedb/src/data/executor/handlers/point/get.rs +++ b/nodedb/src/data/executor/handlers/point/get.rs @@ -66,7 +66,6 @@ impl CoreLoop { collection, ); - let bitemporal = self.is_bitemporal(database_id, tid, collection); let is_temporal_read = system_as_of_ms.is_some() || valid_at_ms.is_some(); // Fetch data from cache or storage. Temporal reads bypass the @@ -106,13 +105,7 @@ impl CoreLoop { if let Some(data) = cached { data } else { - let res = if bitemporal { - self.sparse - .versioned_get_current(database_id, tid, collection, &storage_key) - } else { - self.sparse.get(database_id, tid, collection, &storage_key) - }; - match res { + match self.current_row_body(database_id, tid, collection, &storage_key) { Ok(Some(data)) => { self.doc_cache .put(database_id, tid, collection, &storage_key, &data); diff --git a/nodedb/src/data/executor/handlers/point/mod.rs b/nodedb/src/data/executor/handlers/point/mod.rs index 5d1fe6785..6485a18d0 100644 --- a/nodedb/src/data/executor/handlers/point/mod.rs +++ b/nodedb/src/data/executor/handlers/point/mod.rs @@ -10,6 +10,7 @@ pub mod apply_delete; pub mod apply_put; pub mod delete; +pub mod delete_version; pub mod get; pub mod insert; pub mod overlay_lookup; diff --git a/nodedb/src/data/executor/handlers/point/update/exec.rs b/nodedb/src/data/executor/handlers/point/update/exec.rs index 109b15dc7..3d336a74b 100644 --- a/nodedb/src/data/executor/handlers/point/update/exec.rs +++ b/nodedb/src/data/executor/handlers/point/update/exec.rs @@ -140,13 +140,7 @@ impl CoreLoop { 0 }; let database_id = task.request.database_id.as_u64(); - let get_result = if bitemporal { - self.sparse - .versioned_get_current(database_id, tid, collection, &storage_key) - } else { - self.sparse.get(database_id, tid, collection, &storage_key) - }; - match get_result { + match self.current_row_body(database_id, tid, collection, &storage_key) { Ok(Some(current_bytes)) => { // A period lock gates both images of an update: the PRE-image, // because a closed period must reject any edit to a row it @@ -157,16 +151,14 @@ impl CoreLoop { // both refuse. if let Some(config) = self.doc_configs.get(&config_key) && let Some(ref pl) = config.enforcement.period_lock - && let Err(e) = - crate::data::executor::enforcement::period_lock::check_period_lock( - &self.sparse, - database_id, - tid, - collection, - ¤t_bytes, - pl, - resolved_sum_targets, - ) + && let Err(e) = self.check_period_lock_at( + database_id, + tid, + collection, + ¤t_bytes, + pl, + resolved_sum_targets, + ) { return self.response_error(task, e); } @@ -199,16 +191,14 @@ impl CoreLoop { // even when the pre-image lived in an open one. if let Some(config) = self.doc_configs.get(&config_key) && let Some(ref pl) = config.enforcement.period_lock - && let Err(e) = - crate::data::executor::enforcement::period_lock::check_period_lock( - &self.sparse, - database_id, - tid, - collection, - &updated_bytes, - pl, - resolved_sum_targets, - ) + && let Err(e) = self.check_period_lock_at( + database_id, + tid, + collection, + &updated_bytes, + pl, + resolved_sum_targets, + ) { return self.response_error(task, e); } diff --git a/nodedb/src/data/executor/handlers/provider_scan.rs b/nodedb/src/data/executor/handlers/provider_scan.rs index 65f244bf3..998a0aae7 100644 --- a/nodedb/src/data/executor/handlers/provider_scan.rs +++ b/nodedb/src/data/executor/handlers/provider_scan.rs @@ -79,24 +79,11 @@ impl CoreLoop { }; if !predicates.is_empty() { // `Vec::retain`'s closure must return `bool`, so an - // evaluation error is captured via this side-channel and - // checked once the retain finishes. - let predicate_err: std::cell::RefCell> = - std::cell::RefCell::new(None); - rows.retain(|row| { - if predicate_err.borrow().is_some() { - return true; - } - match ScanFilter::all_match_binary(&predicates, row) { - Ok(keep) => keep, - Err(e) => { - predicate_err.replace(Some(e)); - true - } - } - }); - if let Some(e) = predicate_err.take() { - return self.response_error(task, ErrorCode::from(e)); + // evaluation error is held and returned once the retain ends. + let held = crate::data::executor::core_loop::filter_match::HeldEvalError::default(); + rows.retain(|row| held.check(|| ScanFilter::all_match_binary(&predicates, row))); + if let Err(e) = held.finish::<_, ErrorCode>(()) { + return self.response_error(task, e); } } } diff --git a/nodedb/src/data/executor/handlers/rag_key.rs b/nodedb/src/data/executor/handlers/rag_key.rs new file mode 100644 index 000000000..226ea2698 --- /dev/null +++ b/nodedb/src/data/executor/handlers/rag_key.rs @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The key the legs of a GraphRAG fusion fuse on. +//! +//! A row named by a graph node fuses under that name. A row with a surrogate +//! and no graph node fuses under the surrogate. A vector entry with no +//! surrogate fuses with nothing. Equality keeps the three apart, so a key +//! `"7"` never meets surrogate 7. Ordering compares the rendered text first, +//! so equal RRF scores break in the order of the rendered strings. + +use std::borrow::Cow; +use std::cmp::Ordering; + +use nodedb_types::Surrogate; + +/// Prefix of the rendered key of a vector entry with no surrogate. +pub(crate) const UNBOUND_PREFIX: &str = "__unbound_"; + +/// One fusion leg hit, in the key space RRF fuses on. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub(crate) enum RagKey { + /// A graph node name: the row's client key. + Node(String), + /// A row with no graph node name. The response boundary names it. + Row(Surrogate), + /// A vector entry with no surrogate, keyed by its HNSW entry id. + Unbound(u32), +} + +impl RagKey { + /// The key of a hit carrying `surrogate`: the graph node it names, else + /// the surrogate. + pub(crate) fn of_hit(name: Option<&str>, surrogate: Surrogate) -> Self { + match name { + Some(name) => Self::Node(name.to_owned()), + None => Self::Row(surrogate), + } + } + + /// The text this key renders as before the response boundary names it. + pub(crate) fn rendered(&self) -> Cow<'_, str> { + match self { + Self::Node(name) => Cow::Borrowed(name), + Self::Row(surrogate) => Cow::Owned(surrogate.as_u32().to_string()), + Self::Unbound(entry) => Cow::Owned(format!("{UNBOUND_PREFIX}{entry}")), + } + } + + /// The bytes of [`Self::rendered`], with no allocation: a node name + /// borrows itself, and a number writes into `buf`. + fn rendered_bytes<'a>(&'a self, buf: &'a mut [u8; RENDER_CAP]) -> &'a [u8] { + match self { + Self::Node(name) => name.as_bytes(), + Self::Row(surrogate) => write_decimal(buf, 0, surrogate.as_u32()), + Self::Unbound(entry) => { + let prefix = UNBOUND_PREFIX.as_bytes(); + buf[..prefix.len()].copy_from_slice(prefix); + write_decimal(buf, prefix.len(), *entry) + } + } + } + + fn variant_rank(&self) -> u8 { + match self { + Self::Node(_) => 0, + Self::Row(_) => 1, + Self::Unbound(_) => 2, + } + } +} + +/// Room for the longest numeric rendering: the unbound prefix and the ten +/// digits of `u32::MAX`. +const RENDER_CAP: usize = UNBOUND_PREFIX.len() + 10; + +/// Write the decimal digits of `n` into `buf` from `start`. Returns +/// `buf[..start + digits]`. +fn write_decimal(buf: &mut [u8; RENDER_CAP], start: usize, n: u32) -> &[u8] { + let mut digits = [0u8; 10]; + let mut len = 0; + let mut rest = n; + loop { + digits[len] = b'0' + (rest % 10) as u8; + len += 1; + rest /= 10; + if rest == 0 { + break; + } + } + for (offset, digit) in digits[..len].iter().rev().enumerate() { + buf[start + offset] = *digit; + } + &buf[..start + len] +} + +/// Byte order of the rendered texts, the same order `str` comparison gives, +/// then the variant to keep equal texts apart. +impl Ord for RagKey { + fn cmp(&self, other: &Self) -> Ordering { + let mut mine = [0u8; RENDER_CAP]; + let mut theirs = [0u8; RENDER_CAP]; + self.rendered_bytes(&mut mine) + .cmp(other.rendered_bytes(&mut theirs)) + .then_with(|| self.variant_rank().cmp(&other.variant_rank())) + } +} + +impl PartialOrd for RagKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ord_matches_rendered_text_order() { + let mut keys = [ + RagKey::Node("b".into()), + RagKey::Row(Surrogate::new(10)), + RagKey::Row(Surrogate::new(9)), + RagKey::Unbound(3), + RagKey::Node("10a".into()), + ]; + keys.sort(); + let rendered: Vec = keys.iter().map(|k| k.rendered().into_owned()).collect(); + let mut expected = rendered.clone(); + expected.sort(); + assert_eq!(rendered, expected); + } + + /// Mixed digit counts sort as text ("10" before "9"), and a node name + /// equal to a decimal sorts beside it, apart from it. + #[test] + fn ord_matches_rendered_text_order_across_digit_counts() { + let mut keys = vec![ + RagKey::Row(Surrogate::new(9)), + RagKey::Row(Surrogate::new(10)), + RagKey::Row(Surrogate::new(u32::MAX)), + RagKey::Row(Surrogate::new(0)), + RagKey::Node("9".into()), + RagKey::Node("10".into()), + RagKey::Unbound(9), + RagKey::Unbound(10), + RagKey::Unbound(u32::MAX), + ]; + keys.sort(); + let rendered: Vec = keys.iter().map(|k| k.rendered().into_owned()).collect(); + let mut expected = rendered.clone(); + expected.sort(); + assert_eq!(rendered, expected); + let nine = keys + .iter() + .position(|k| *k == RagKey::Node("9".into())) + .expect("node 9"); + assert_eq!(keys[nine + 1], RagKey::Row(Surrogate::new(9))); + } + + #[test] + fn decimal_node_never_equals_an_equal_surrogate() { + assert_ne!(RagKey::Node("7".into()), RagKey::Row(Surrogate::new(7))); + assert_ne!( + RagKey::Node("7".into()).cmp(&RagKey::Row(Surrogate::new(7))), + Ordering::Equal + ); + } + + #[test] + fn unbound_renders_the_unbound_sentinel() { + assert_eq!(RagKey::Unbound(4).rendered(), "__unbound_4"); + } + + #[test] + fn of_hit_prefers_the_graph_node_name() { + let s = Surrogate::new(5); + assert_eq!(RagKey::of_hit(Some("a"), s), RagKey::Node("a".into())); + assert_eq!(RagKey::of_hit(None, s), RagKey::Row(s)); + } +} diff --git a/nodedb/src/data/executor/handlers/recursive.rs b/nodedb/src/data/executor/handlers/recursive.rs index c3b1ed24e..96fe909a0 100644 --- a/nodedb/src/data/executor/handlers/recursive.rs +++ b/nodedb/src/data/executor/handlers/recursive.rs @@ -104,8 +104,9 @@ impl CoreLoop { collection, ); - // Scan all documents once (used for both base and recursive steps). - let all_docs = match self.sparse.scan_documents( + // Scan all documents once (used for both base and recursive steps), + // each at its current version. + let all_docs = match self.current_rows( task.request.database_id.as_u64(), tid, collection, diff --git a/nodedb/src/data/executor/handlers/spatial/rtree_scan.rs b/nodedb/src/data/executor/handlers/spatial/rtree_scan.rs index 498ca9a25..84c181800 100644 --- a/nodedb/src/data/executor/handlers/spatial/rtree_scan.rs +++ b/nodedb/src/data/executor/handlers/spatial/rtree_scan.rs @@ -230,7 +230,8 @@ impl CoreLoop { // to standard msgpack maps that `decode_document_value` and // `extract_geometry` read identically. let sparse_result = match storage_key { - Some(key) => self.sparse.get(database_id, tid, collection, &key), + // At its current version, so a bitemporal collection resolves. + Some(key) => self.current_row_body(database_id, tid, collection, &key), None => Ok(None), }; let doc = match sparse_result { diff --git a/nodedb/src/data/executor/handlers/text_rows.rs b/nodedb/src/data/executor/handlers/text_rows.rs index 1d5af69bd..d6747d1bd 100644 --- a/nodedb/src/data/executor/handlers/text_rows.rs +++ b/nodedb/src/data/executor/handlers/text_rows.rs @@ -17,13 +17,8 @@ use crate::data::executor::handlers::transaction::overlay::Staged; use crate::data::executor::row_shape::sparse_row_to_doc; use crate::data::executor::sparse_body_format::SparseBodyFormatRef; use crate::data::executor::task::ExecutionTask; -use crate::engine::sparse::btree_versioned::VersionedScanParams; -use crate::engine::sparse::scan_stop::never_stop; use crate::types::TenantId; -/// Rows read per page of a bitemporal collection's current versions. -const BITEMPORAL_PAGE_ROWS: usize = 1024; - /// The row predicates a full-text read applies before ranking: the residual /// WHERE filters and the RLS filters. pub(in crate::data::executor) struct TextRowGate { @@ -73,70 +68,6 @@ pub(in crate::data::executor) fn text_row_image( } impl CoreLoop { - /// The current version of one base row: the versioned table's current - /// version for a bitemporal collection, the document table otherwise. - pub(in crate::data::executor) fn text_base_body( - &self, - database_id: u64, - tid: u64, - collection: &str, - key: &StorageKey, - ) -> crate::Result>> { - if self.is_bitemporal(database_id, tid, collection) { - self.sparse - .versioned_get_current(database_id, tid, collection, key) - } else { - self.sparse.get(database_id, tid, collection, key) - } - } - - /// Visit the current version of every base row of `collection`, from the - /// tables [`Self::text_base_body`] reads, until `f` breaks. A bitemporal - /// collection is read a page at a time, so at most one page of rows is in - /// memory. - pub(in crate::data::executor) fn for_each_text_base_row( - &self, - database_id: u64, - tid: u64, - collection: &str, - mut f: F, - ) -> crate::Result<()> - where - F: FnMut(&StorageKey, &[u8]) -> crate::Result>, - { - if !self.is_bitemporal(database_id, tid, collection) { - return self - .sparse - .scan_documents_while(database_id, tid, collection, f); - } - let mut after: Option = None; - loop { - let page = self.sparse.versioned_scan_as_of_after( - VersionedScanParams { - database_id, - tenant: tid, - coll: collection, - sys_cutoff_ms: None, - valid_at_ms: None, - limit: BITEMPORAL_PAGE_ROWS, - }, - after.as_ref(), - &|_, _| true, - &never_stop, - )?; - let full = page.len() >= BITEMPORAL_PAGE_ROWS; - for (key, body) in &page { - if f(key, body)?.is_break() { - return Ok(()); - } - } - match page.into_iter().next_back() { - Some((key, _)) if full => after = Some(key), - _ => return Ok(()), - } - } - } - /// Surrogates of `collection` whose row passes `gate`, judged before /// ranking. Staged rows of the issuing transaction are judged on their /// staged body. A staged tombstone or TRUNCATE removes base rows. `None` @@ -175,7 +106,7 @@ impl CoreLoop { let mut eligible = SurrogateBitmap::new(); if overlay.is_none_or(|o| o.base_visible(&coll_key)) { - self.for_each_text_base_row(database_id.as_u64(), tid, collection, |key, bytes| { + self.for_each_current_row(database_id.as_u64(), tid, collection, |key, bytes| { if admits(key, bytes)? { eligible.insert(key.surrogate()); } diff --git a/nodedb/src/data/executor/handlers/text_search.rs b/nodedb/src/data/executor/handlers/text_search.rs index 931614a94..77d06e9ef 100644 --- a/nodedb/src/data/executor/handlers/text_search.rs +++ b/nodedb/src/data/executor/handlers/text_search.rs @@ -210,7 +210,7 @@ impl CoreLoop { for (surrogate, score, fuzzy) in hits { let storage_key = nodedb_types::StorageKey::for_surrogate(surrogate); let bytes_opt = self.overlay_or_base_body(txn_id, &coll_key, &storage_key, || { - self.text_base_body(database_id, tid, collection, &storage_key) + self.current_row_body(database_id, tid, collection, &storage_key) })?; // A surrogate with no body was indexed for FTS without a document // write (FtsIndex frames synced from Lite). Its row is the diff --git a/nodedb/src/data/executor/handlers/text_search_scan.rs b/nodedb/src/data/executor/handlers/text_search_scan.rs index ad844236e..a471cb677 100644 --- a/nodedb/src/data/executor/handlers/text_search_scan.rs +++ b/nodedb/src/data/executor/handlers/text_search_scan.rs @@ -239,21 +239,15 @@ impl CoreLoop { let mut stopped = false; if overlay.is_none_or(|o| o.base_visible(&coll_key)) { - self.for_each_text_base_row( - database_id.as_u64(), - p.tid, - p.collection, - |key, bytes| { - // A row the transaction staged is emitted from its staged body. - if overlay.is_some_and(|o| o.get(&coll_key, key.surrogate().as_u32()).is_some()) - { - return Ok(ControlFlow::Continue(())); - } - let flow = emit(key, bytes)?; - stopped = flow.is_break(); - Ok(flow) - }, - )?; + self.for_each_current_row(database_id.as_u64(), p.tid, p.collection, |key, bytes| { + // A row the transaction staged is emitted from its staged body. + if overlay.is_some_and(|o| o.get(&coll_key, key.surrogate().as_u32()).is_some()) { + return Ok(ControlFlow::Continue(())); + } + let flow = emit(key, bytes)?; + stopped = flow.is_break(); + Ok(flow) + })?; } if let Some(overlay) = overlay && !stopped diff --git a/nodedb/src/data/executor/handlers/transaction/overlay/merge.rs b/nodedb/src/data/executor/handlers/transaction/overlay/merge.rs index f65e1a1aa..d8c17a4c7 100644 --- a/nodedb/src/data/executor/handlers/transaction/overlay/merge.rs +++ b/nodedb/src/data/executor/handlers/transaction/overlay/merge.rs @@ -322,8 +322,8 @@ impl CoreLoop { let target = normalize(value.to_string()); // `value_matches` feeds `Vec::retain` and a plain `for` loop, both of // which need a `bool`, so an undecodable staged body is captured via - // this `Cell` side-channel and checked once both passes finish — - // the same shape `predicate_err` below uses. Treating it as "does not + // this `Cell` side-channel and checked once both passes finish, as + // the residual's held evaluation error is. Treating it as "does not // match" and moving on would drop a staged row from the result with no // indication that a row was dropped. let decode_err: std::cell::Cell> = std::cell::Cell::new(None); @@ -348,30 +348,24 @@ impl CoreLoop { // but not the residual is excluded exactly like it would be from a // base scan result. // `residual_matches` feeds `Vec::retain`/a plain `for` loop below, - // both of which need a `bool`, so a division/modulo-by-zero is - // captured via this `Cell` side-channel and checked once both - // passes finish. - let predicate_err: std::cell::Cell> = - std::cell::Cell::new(None); + // both of which need a `bool`, so a division/modulo-by-zero is held + // and returned once both passes finish. + let held = crate::data::executor::core_loop::filter_match::HeldEvalError::default(); let identity_column = self.identity_column(coll_key.0.as_u64(), coll_key.1.as_u64(), &coll_key.2); let residual_matches = |row_key: &StorageKey, body: &[u8]| -> bool { if residual.is_empty() { return true; } - match matches_with_resolved_schema( - strict_schema, - residual, - row_key, - body, - &identity_column, - ) { - Ok(b) => b, - Err(e) => { - predicate_err.set(Some(e)); - false - } - } + held.check(|| { + matches_with_resolved_schema( + strict_schema, + residual, + row_key, + body, + &identity_column, + ) + }) }; // Base doc IDs' surrogates, so additions don't re-append a row the @@ -414,10 +408,7 @@ impl CoreLoop { if let Some(e) = decode_err.take() { return Err(e); } - if let Some(e) = predicate_err.take() { - return Err(crate::Error::from(e)); - } - Ok(()) + held.finish(()) } /// Resolve the current body for a hex-surrogate `doc_id` in `coll_key`, diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/test_commit.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/test_commit.rs index 73b01ca2b..af132775c 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/test_commit.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/test_commit.rs @@ -10,12 +10,12 @@ use nodedb_physical::physical_plan::PhysicalPlan; use crate::bridge::envelope::{Response, Status}; -use crate::control::wal_replication::transaction_redo::collections::written_collections; use crate::control::wal_replication::transaction_redo::sum_targets::redo_sum_targets; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::task::ExecutionTask; use crate::types::{Lsn, TxnId}; use crate::wal::{RedoRecord, RedoSubRecord}; +use nodedb_physical::physical_plan::written_collections; use nodedb_wal::record::RecordType; use super::entry::CommittedRedo; diff --git a/nodedb/src/data/executor/handlers/transaction/redo_apply/validate.rs b/nodedb/src/data/executor/handlers/transaction/redo_apply/validate.rs index b3609b050..017079bb0 100644 --- a/nodedb/src/data/executor/handlers/transaction/redo_apply/validate.rs +++ b/nodedb/src/data/executor/handlers/transaction/redo_apply/validate.rs @@ -153,7 +153,7 @@ impl CoreLoop { // delete judges whatever is stored, as the delete path does. if !scope.bitemporal || prior.is_some() { run_delete_enforcement( - &self.sparse, + self, scope.database_id, scope.tid, scope.collection, @@ -227,13 +227,7 @@ impl CoreLoop { surrogate: u32, ) -> crate::Result>> { let key = StorageKey::for_surrogate(Surrogate::new(surrogate)); - if scope.bitemporal { - self.sparse - .versioned_get_current(scope.database_id, scope.tid, scope.collection, &key) - } else { - self.sparse - .get(scope.database_id, scope.tid, scope.collection, &key) - } + self.current_row_body(scope.database_id, scope.tid, scope.collection, &key) } } diff --git a/nodedb/src/data/executor/handlers/transaction/resolve/document.rs b/nodedb/src/data/executor/handlers/transaction/resolve/document.rs index 1a0853c54..b082b0258 100644 --- a/nodedb/src/data/executor/handlers/transaction/resolve/document.rs +++ b/nodedb/src/data/executor/handlers/transaction/resolve/document.rs @@ -201,24 +201,15 @@ impl CoreLoop { overlay: &TxnOverlay, coll_key: &(DatabaseId, TenantId, String), ) -> crate::Result)>> { + // Each base row at its current version: the newest version of each + // row on a bitemporal collection, the plain table on any other. let database_id = task.request.database_id.as_u64(); - let bitemporal = self.is_bitemporal(database_id, tid, collection); - let mut rows = Vec::new(); - for key in self.scan_matching_documents(database_id, tid, collection, &[])? { - if overlay.get(coll_key, key.surrogate().as_u32()).is_some() { - continue; - } - let body = if bitemporal { - self.sparse - .versioned_get_current(database_id, tid, collection, &key)? - } else { - self.sparse.get(database_id, tid, collection, &key)? - }; - if let Some(body) = body { - rows.push((key, body)); - } - } - Ok(rows) + self.current_rows_where( + database_id, + tid, + collection, + &|key: &StorageKey, _: &[u8]| overlay.get(coll_key, key.surrogate().as_u32()).is_none(), + ) } } diff --git a/nodedb/src/data/executor/handlers/transaction/resolve/row_changes/keyed.rs b/nodedb/src/data/executor/handlers/transaction/resolve/row_changes/keyed.rs index b803e539c..be5466a9e 100644 --- a/nodedb/src/data/executor/handlers/transaction/resolve/row_changes/keyed.rs +++ b/nodedb/src/data/executor/handlers/transaction/resolve/row_changes/keyed.rs @@ -257,13 +257,9 @@ impl CoreLoop { surrogate: u32, ) -> crate::Result { let key = StorageKey::for_surrogate(Surrogate::new(surrogate)); - let body = if self.is_bitemporal(database_id, tid, collection) { - self.sparse - .versioned_get_current(database_id, tid, collection, &key)? - } else { - self.sparse.get(database_id, tid, collection, &key)? - }; - Ok(body.is_some()) + Ok(self + .current_row_body(database_id, tid, collection, &key)? + .is_some()) } } diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/dispatch.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/dispatch.rs index 61d9bb114..9e78ff8f5 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/dispatch.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/dispatch.rs @@ -9,9 +9,7 @@ use nodedb_types::RowIdentity; use super::constraint::OverlayPk; use super::context::StageCtx; use super::stage_returning::{StageReturning, StagedImage}; -use super::{ - StageBalanceDeltaParams, StageBulkDeleteParams, StageBulkUpdateParams, StageSpatialInsertParams, -}; +use super::{StageBalanceDeltaParams, StageSpatialInsertParams}; use crate::bridge::envelope::{ErrorCode, PhysicalPlan, Response}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::handlers::transaction::overlay::{MAX_TXN_OVERLAY_BYTES, Staged}; @@ -35,21 +33,13 @@ impl CoreLoop { }, ); }; - // Every core that stages a write for this transaction holds its - // overlay, even when the write stages no row. COMMIT's resolve refuses - // a transaction with staged writes whose overlay is missing. The - // overlay tags what a trigger body stages by this request's source. - let database_id = task.request.database_id; - let tenant_id = crate::types::TenantId::new(tid); - let collections = - crate::control::wal_replication::transaction_redo::collections::written_collections( - std::slice::from_ref(plan), - ); - self.txn_overlay_mut(txn_id).begin_staging( + // The overlay tags what a trigger body stages by this request's source. + self.begin_plan_staging( + txn_id, task.request.event_source, - collections - .into_iter() - .map(|collection| (database_id, tenant_id, collection)), + task.request.database_id, + crate::types::TenantId::new(tid), + plan, ); let doc_op = match plan { @@ -268,56 +258,10 @@ impl CoreLoop { }, ) } - // Predicate UPDATE staged like a point update, resolved against - // base ∪ overlay. A RETURNING spec projects the staged - // post-images. - DocumentOp::BulkUpdate { - collection, - filters, - updates, - returning, - ollp_predicted_surrogates: _, - ollp_predicted_edges: _, - rls_filters, - rls_write_check, - // Staged post-images become concrete point ops at commit. - resolved_sum_targets: _, - declared_primary_key, - } => self.stage_bulk_update(StageBulkUpdateParams { - task, - tid, - txn_id, - collection: collection.as_str(), - filter_bytes: filters, - updates, - rls_write_check, - declared_primary_key: declared_primary_key.as_deref(), - returning: StageReturning::of(returning, rls_filters), - }), - - // Predicate DELETE staged like a point delete, resolved against - // base ∪ overlay. A RETURNING spec projects the removed rows. - DocumentOp::BulkDelete { - collection, - filters, - returning, - ollp_predicted_surrogates: _, - ollp_predicted_edges: _, - rls_filters, - rls_write_check, - // See the `BulkUpdate` arm: staging carries no delta. - resolved_sum_targets: _, - declared_primary_key, - } => self.stage_bulk_delete(StageBulkDeleteParams { - task, - tid, - txn_id, - collection: collection.as_str(), - filter_bytes: filters, - rls_write_check, - declared_primary_key: declared_primary_key.as_deref(), - returning: StageReturning::of(returning, rls_filters), - }), + // Predicate UPDATE and DELETE, resolved against base ∪ overlay. + DocumentOp::BulkUpdate { .. } | DocumentOp::BulkDelete { .. } => { + self.stage_bulk_document_write(task, tid, txn_id, doc_op) + } // `UPSERT INTO`: resolve the current body under base ∪ overlay, // mirroring `execute_upsert`. A RETURNING spec projects the row diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/mod.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/mod.rs index 45ac2d917..3e8a481b3 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/mod.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/mod.rs @@ -16,8 +16,10 @@ mod body; mod constraint; mod context; mod dispatch; +mod plan_staging; mod stage_array; mod stage_bulk_delete; +mod stage_bulk_dispatch; mod stage_bulk_update; mod stage_columnar; mod stage_columnar_base_key; diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/plan_staging.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/plan_staging.rs new file mode 100644 index 000000000..86e8f5448 --- /dev/null +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/plan_staging.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Opening one write plan's staging in a transaction overlay. +//! +//! Every core that stages a plan for a transaction holds its overlay, even +//! when the plan stages no row: resolve refuses a transaction with staged +//! writes whose overlay is missing. The overlay tags what the plan stages +//! with the source it stages under, so a trigger body's rows commit as the +//! body's. + +use nodedb_physical::physical_plan::{PhysicalPlan, written_collections}; + +use crate::data::executor::core_loop::CoreLoop; +use crate::event::EventSource; +use crate::types::{DatabaseId, TenantId, TxnId}; + +impl CoreLoop { + /// Open `plan`'s staging in transaction `txn_id`'s overlay under + /// `source`, for every collection the plan writes. + pub(in crate::data::executor) fn begin_plan_staging( + &mut self, + txn_id: TxnId, + source: EventSource, + database_id: DatabaseId, + tenant_id: TenantId, + plan: &PhysicalPlan, + ) { + let collections = written_collections(std::slice::from_ref(plan)); + self.txn_overlay_mut(txn_id).begin_staging( + source, + collections + .into_iter() + .map(|collection| (database_id, tenant_id, collection)), + ); + } +} diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_bulk_delete.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_bulk_delete.rs index 7477b5910..7add240b1 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_bulk_delete.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_bulk_delete.rs @@ -7,7 +7,7 @@ //! //! Mirrors point `DELETE` staging in `dispatch.rs` (`stage_point_delete`): //! each row in the BASE ∪ OVERLAY matching set (resolved via -//! [`CoreLoop::merge_overlay_into_scan`]) is recorded as a staged tombstone. +//! [`CoreLoop::txn_matching_rows`]) is recorded as a staged tombstone. //! Nothing is written durably here — cascades (inverted index, secondary //! indexes, graph edges) run only at COMMIT replay through the real apply //! path, exactly as a staged point delete defers its cascade today. @@ -17,8 +17,8 @@ use nodedb_types::StorageKey; use super::body::stored_row_identity; use super::stage_returning::{StageReturning, StagedRowsOf}; use crate::bridge::envelope::{ErrorCode, Response}; -use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::bulk_dml::{TxnPrediction, decode_bulk_filters}; use crate::data::executor::response_codec; use crate::data::executor::task::ExecutionTask; use crate::types::{DatabaseId, TenantId, TxnId}; @@ -39,6 +39,9 @@ pub(in crate::data::executor) struct StageBulkDeleteParams<'a> { pub declared_primary_key: Option<&'a str>, /// The plan's `RETURNING` spec, `None` when it carries none. pub returning: Option>, + /// The rows the Control Plane read at this statement. `None` when the + /// plan carries none. + pub prediction: Option>, } impl CoreLoop { @@ -59,61 +62,28 @@ impl CoreLoop { rls_write_check, declared_primary_key, returning, + prediction, } = params; let database_id = task.request.database_id; let coll_key: (DatabaseId, TenantId, String) = (database_id, TenantId::new(tid), collection.to_string()); - let filters: Vec = if filter_bytes.is_empty() { - Vec::new() - } else { - match zerompk::from_msgpack(filter_bytes) { - Ok(f) => f, - Err(e) => { - return self.response_error( - task, - ErrorCode::Internal { - detail: format!("deserialize filters: {e}"), - }, - ); - } - } - }; - - // BASE matching set: the same scan-and-filter primitive the - // autocommit bulk handler uses, then fetch each matched row's - // current body so the overlay merge can re-check staged puts. - let mut rows = match self.stage_bulk_base_rows( - task, - database_id.as_u64(), - tid, - collection, - &filters, - ) { + // The BASE ∪ OVERLAY matching set, BASE read at each row's current + // version. + let rows = match decode_bulk_filters(filter_bytes) + .and_then(|filters| self.txn_matching_rows(txn_id, &coll_key, &filters)) + { Ok(rows) => rows, - Err(resp) => return resp, + Err(e) => return self.response_error(task, e), }; + // The carried prediction is the read the Control Plane took at this + // statement. Rows that differ refuse the stage before anything stages, + // and the Control Plane reads again. + if let Some(prediction) = prediction + && !prediction.holds(&rows) { - // `merge_overlay_into_scan` takes an infallible - // `Fn(&StorageKey, &[u8]) -> bool` predicate, so a division/modulo- - // by-zero is captured via this `Cell` side-channel and checked once - // the merge returns. - let raw_matches = - self.strict_aware_matcher(database_id.as_u64(), tid, collection, &filters); - let predicate_err: std::cell::Cell> = - std::cell::Cell::new(None); - let matches = |row_key: &StorageKey, body: &[u8]| match raw_matches(row_key, body) { - Ok(b) => b, - Err(e) => { - predicate_err.set(Some(e)); - false - } - }; - self.merge_overlay_into_scan(txn_id, &coll_key, &mut rows, &matches); - if let Some(e) = predicate_err.take() { - return self.response_error(task, ErrorCode::from(e)); - } + return self.response_error(task, ErrorCode::OllpRetryRequired); } // Gate every matched row on the collection's write policy BEFORE any @@ -187,3 +157,81 @@ impl CoreLoop { } } } + +#[cfg(test)] +mod tests { + use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan}; + use nodedb_types::{DatabaseId, QualifiedCollection, Surrogate}; + + use crate::bridge::envelope::{ErrorCode, Status}; + use crate::data::executor::core_loop::CoreLoop; + use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; + use crate::data::executor::doc_format; + use crate::data::executor::task::ExecutionTask; + use crate::types::{TenantId, TxnId}; + + fn delete_all(predicted: Option>) -> PhysicalPlan { + PhysicalPlan::Document(DocumentOp::BulkDelete { + collection: QualifiedCollection::new(DatabaseId::DEFAULT, "orders"), + filters: Vec::new(), + returning: None, + ollp_predicted_surrogates: predicted, + ollp_predicted_edges: None, + rls_filters: Vec::new(), + rls_write_check: nodedb_types::RlsWriteCheck::NoPolicyApplies, + resolved_sum_targets: Vec::new(), + declared_primary_key: None, + }) + } + + fn seed(core: &mut CoreLoop, task: &ExecutionTask, surrogate: u32) { + let body = doc_format::encode_to_msgpack(&serde_json::json!({"a": "1"})); + core.sparse + .put( + task.request.database_id.as_u64(), + task.request.tenant_id.as_u64(), + "orders", + &nodedb_types::StorageKey::for_surrogate(Surrogate::new(surrogate)), + &body, + ) + .expect("seed row"); + } + + /// A predicate delete whose carried prediction misses a matching row + /// refuses with `OllpRetryRequired` and stages no tombstone. + #[test] + fn stale_prediction_refuses_the_stage_and_stages_nothing() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _tx, _rx): (CoreLoop, _, _) = make_core_with_dir(dir.path()); + let txn_id = TxnId::new(7); + let mut request = make_default_task().request; + request.txn_id = Some(txn_id); + let task = ExecutionTask::new(request); + seed(&mut core, &task, 1); + seed(&mut core, &task, 2); + let tid = task.request.tenant_id.as_u64(); + + let plan = delete_all(Some(vec![1])); + let refused = core.execute_stage_write(&task, tid, &plan); + assert_eq!(refused.status, Status::Error); + assert_eq!( + refused.error_code.as_deref(), + Some(&ErrorCode::OllpRetryRequired) + ); + let coll_key = ( + task.request.database_id, + TenantId::new(tid), + "orders".to_string(), + ); + let staged = core + .txn_overlays + .get(&txn_id) + .map(|overlay| overlay.iter_for_collection(&coll_key).count()) + .unwrap_or(0); + assert_eq!(staged, 0, "a refused stage stages no tombstone"); + + let plan = delete_all(Some(vec![1, 2])); + let accepted = core.execute_stage_write(&task, tid, &plan); + assert_eq!(accepted.status, Status::Ok, "{:?}", accepted.error_code); + } +} diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_bulk_dispatch.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_bulk_dispatch.rs new file mode 100644 index 000000000..730870cce --- /dev/null +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_bulk_dispatch.rs @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Statement-time staging dispatch for a predicate `UPDATE` or `DELETE` +//! (`DocumentOp::BulkUpdate` / `BulkDelete`). +//! +//! The plan carries the prediction the Control Plane read at this statement. +//! The stage checks it against the rows it matches, and refuses with +//! `OllpRetryRequired` before staging anything when they differ. + +use nodedb_physical::physical_plan::DocumentOp; + +use super::stage_returning::StageReturning; +use super::{StageBulkDeleteParams, StageBulkUpdateParams}; +use crate::bridge::envelope::{ErrorCode, Response}; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::bulk_dml::TxnPrediction; +use crate::data::executor::task::ExecutionTask; +use crate::types::TxnId; + +impl CoreLoop { + /// Stage one predicate write. Any other document op is refused: it never + /// routes here. + pub(super) fn stage_bulk_document_write( + &mut self, + task: &ExecutionTask, + tid: u64, + txn_id: TxnId, + op: &DocumentOp, + ) -> Response { + match op { + // Staged like a point update. A RETURNING spec projects the staged + // post-images. + DocumentOp::BulkUpdate { + collection, + filters, + updates, + returning, + ollp_predicted_surrogates, + ollp_predicted_edges, + rls_filters, + rls_write_check, + // Staged post-images become concrete point ops at commit. + resolved_sum_targets: _, + declared_primary_key, + } => self.stage_bulk_update(StageBulkUpdateParams { + task, + tid, + txn_id, + collection: collection.as_str(), + filter_bytes: filters, + updates, + rls_write_check, + declared_primary_key: declared_primary_key.as_deref(), + returning: StageReturning::of(returning, rls_filters), + prediction: TxnPrediction::of(ollp_predicted_surrogates, ollp_predicted_edges), + }), + // Staged like a point delete. A RETURNING spec projects the + // removed rows. + DocumentOp::BulkDelete { + collection, + filters, + returning, + ollp_predicted_surrogates, + ollp_predicted_edges, + rls_filters, + rls_write_check, + // See the `BulkUpdate` arm: staging carries no delta. + resolved_sum_targets: _, + declared_primary_key, + } => self.stage_bulk_delete(StageBulkDeleteParams { + task, + tid, + txn_id, + collection: collection.as_str(), + filter_bytes: filters, + rls_write_check, + declared_primary_key: declared_primary_key.as_deref(), + returning: StageReturning::of(returning, rls_filters), + prediction: TxnPrediction::of(ollp_predicted_surrogates, ollp_predicted_edges), + }), + DocumentOp::PointGet { .. } + | DocumentOp::PointPut { .. } + | DocumentOp::PointInsert { .. } + | DocumentOp::PointDelete { .. } + | DocumentOp::PointUpdate { .. } + | DocumentOp::Upsert { .. } + | DocumentOp::BatchUpsert { .. } + | DocumentOp::BatchInsert { .. } + | DocumentOp::Scan { .. } + | DocumentOp::RangeScan { .. } + | DocumentOp::Register { .. } + | DocumentOp::IndexLookup { .. } + | DocumentOp::IndexedFetch { .. } + | DocumentOp::DropIndex { .. } + | DocumentOp::BackfillIndex { .. } + | DocumentOp::Truncate { .. } + | DocumentOp::EstimateCount { .. } + | DocumentOp::InsertSelect { .. } + | DocumentOp::UpdateFromJoin { .. } + | DocumentOp::Merge { .. } + | DocumentOp::ResolveWrite(_) + | DocumentOp::MaterializeScan { .. } + | DocumentOp::ResolvedWrite { .. } + | DocumentOp::ApplyBalanceDelta { .. } => self.response_error( + task, + ErrorCode::Internal { + detail: "a document op that is not a predicate write reached predicate \ + staging" + .into(), + }, + ), + } + } +} diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_bulk_update.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_bulk_update.rs index a381b958d..84ab70996 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_bulk_update.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_bulk_update.rs @@ -6,7 +6,7 @@ //! identically; the reply projects those post-images per the clause. //! //! Mirrors the point-write staging in `dispatch.rs`: the matched rows are -//! evaluated against BASE ∪ OVERLAY (via [`CoreLoop::merge_overlay_into_scan`]) +//! evaluated against BASE ∪ OVERLAY (via [`CoreLoop::txn_matching_rows`]) //! so a same-transaction earlier write to a row is observed, each matched //! row's update is applied with the SAME per-row primitive point `UPDATE` //! uses ([`CoreLoop::stage_apply_update`]), and the result is staged into the @@ -14,14 +14,14 @@ //! replay remains the sole durable apply. use nodedb_physical::physical_plan::UpdateValue; -use nodedb_types::{RowIdentity, StorageKey}; +use nodedb_types::RowIdentity; use super::body::stored_row_identity; use super::constraint::StagedStatement; use super::stage_returning::{StageReturning, StagedRowsOf}; use crate::bridge::envelope::{ErrorCode, Response}; -use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::bulk_dml::{TxnPrediction, decode_bulk_filters}; use crate::data::executor::handlers::generated; use crate::data::executor::handlers::transaction::overlay::MAX_TXN_OVERLAY_BYTES; use crate::data::executor::response_codec; @@ -44,6 +44,9 @@ pub(in crate::data::executor) struct StageBulkUpdateParams<'a> { pub declared_primary_key: Option<&'a str>, /// The plan's `RETURNING` spec, `None` when it carries none. pub returning: Option>, + /// The rows the Control Plane read at this statement. `None` when the + /// plan carries none. + pub prediction: Option>, } impl CoreLoop { @@ -65,6 +68,7 @@ impl CoreLoop { rls_write_check, declared_primary_key, returning, + prediction, } = params; let database_id = task.request.database_id; let coll_key: (DatabaseId, TenantId, String) = @@ -80,60 +84,22 @@ impl CoreLoop { return self.response_error(task, e); } - let filters: Vec = if filter_bytes.is_empty() { - Vec::new() - } else { - match zerompk::from_msgpack(filter_bytes) { - Ok(f) => f, - Err(e) => { - return self.response_error( - task, - ErrorCode::Internal { - detail: format!("deserialize filters: {e}"), - }, - ); - } - } - }; - - // BASE matching set: the same scan-and-filter primitive the - // autocommit bulk handler uses, then fetch each matched row's - // current body (also the same lookup the autocommit path uses). - let mut rows = match self.stage_bulk_base_rows( - task, - database_id.as_u64(), - tid, - collection, - &filters, - ) { + // The BASE ∪ OVERLAY matching set: the transaction's own staged + // writes fold into BASE, read at each row's current version. + let rows = match decode_bulk_filters(filter_bytes) + .and_then(|filters| self.txn_matching_rows(txn_id, &coll_key, &filters)) + { Ok(rows) => rows, - Err(resp) => return resp, + Err(e) => return self.response_error(task, e), }; - // Fold the transaction's own staged writes into the base result: - // drops tombstoned rows, re-checks staged puts against the predicate - // (an earlier staged update may have moved a row in or out), and - // appends overlay-only rows that now match. + // The carried prediction is the read the Control Plane took at this + // statement. Rows that differ refuse the stage before anything stages, + // and the Control Plane reads again. + if let Some(prediction) = prediction + && !prediction.holds(&rows) { - // `merge_overlay_into_scan` takes an infallible - // `Fn(&StorageKey, &[u8]) -> bool` predicate, so a division/modulo- - // by-zero is captured via this `Cell` side-channel and checked once - // the merge returns. - let raw_matches = - self.strict_aware_matcher(database_id.as_u64(), tid, collection, &filters); - let predicate_err: std::cell::Cell> = - std::cell::Cell::new(None); - let matches = |row_key: &StorageKey, body: &[u8]| match raw_matches(row_key, body) { - Ok(b) => b, - Err(e) => { - predicate_err.set(Some(e)); - false - } - }; - self.merge_overlay_into_scan(txn_id, &coll_key, &mut rows, &matches); - if let Some(e) = predicate_err.take() { - return self.response_error(task, crate::Error::from(e)); - } + return self.response_error(task, ErrorCode::OllpRetryRequired); } let strict_schema = self.resolve_strict_schema(database_id.as_u64(), tid, collection); @@ -231,31 +197,6 @@ impl CoreLoop { } } - /// Resolve the BASE matching set for a bulk predicate stage (shared by - /// `stage_bulk_update` and `stage_bulk_delete`): the same - /// scan-and-filter primitive the autocommit bulk handlers use - /// ([`CoreLoop::scan_matching_documents`]), then each matched row's - /// current body via the same per-row lookup the autocommit path uses. - pub(super) fn stage_bulk_base_rows( - &self, - task: &ExecutionTask, - database_id: u64, - tid: u64, - collection: &str, - filters: &[ScanFilter], - ) -> Result)>, Response> { - let matching_ids = self - .scan_matching_documents(database_id, tid, collection, filters) - .map_err(|e| self.response_error(task, e))?; - let mut rows: Vec<(StorageKey, Vec)> = Vec::with_capacity(matching_ids.len()); - for key in matching_ids { - if let Ok(Some(bytes)) = self.sparse.get(database_id, tid, collection, &key) { - rows.push((key, bytes)); - } - } - Ok(rows) - } - /// Stage a bulk-matched row's new body after enforcing the same /// per-transaction overlay memory cap point writes enforce /// (`stage_put_capped` in `dispatch.rs`): reject with the identical diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_current_body.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_current_body.rs index 8cb6f01ab..bf9b0e641 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_current_body.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_current_body.rs @@ -27,17 +27,7 @@ impl CoreLoop { None if !self.stage_base_visible(ctx.txn_id, &ctx.coll_key) => Ok(None), None => { let storage_key = StorageKey::for_surrogate(ctx.surrogate); - if self.is_bitemporal(ctx.database_id, ctx.tid, ctx.collection) { - self.sparse.versioned_get_current( - ctx.database_id, - ctx.tid, - ctx.collection, - &storage_key, - ) - } else { - self.sparse - .get(ctx.database_id, ctx.tid, ctx.collection, &storage_key) - } + self.current_row_body(ctx.database_id, ctx.tid, ctx.collection, &storage_key) } } } diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs index 7c5251563..a1b0f3c88 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_kv_predicate.rs @@ -21,6 +21,7 @@ use super::stage_kv::kv_row_identity; use crate::bridge::envelope::Response; use crate::bridge::scan_filter::{ScanFilter, decode_scan_filters}; use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::core_loop::filter_match::HeldEvalError; use crate::data::executor::handlers::kv::field_compute::merge_field_updates; use crate::data::executor::response_codec; use crate::data::executor::scan_normalize::kv_row_to_doc; @@ -211,27 +212,18 @@ impl CoreLoop { .map_err(|e| self.response_error(task, e))?; // `merge_kv_overlay_into_scan` takes an infallible predicate, so a - // division/modulo-by-zero is captured via this `Cell` side-channel - // and checked once the merge returns. - let predicate_err: std::cell::Cell> = - std::cell::Cell::new(None); + // division/modulo-by-zero is held and returned once the merge ends. + let held = HeldEvalError::default(); let matches = |key: &[u8], value: &[u8]| { if filters.is_empty() { return true; } - let (_key_str, row) = kv_row_to_doc(key, value); - match ScanFilter::all_match_binary(&filters, &row) { - Ok(b) => b, - Err(e) => { - predicate_err.set(Some(e)); - false - } - } + held.check(|| ScanFilter::all_match_binary(&filters, &kv_row_to_doc(key, value).1)) }; self.merge_kv_overlay_into_scan(txn_id, coll_key, &mut rows, &matches); - if let Some(e) = predicate_err.take() { - return Err(self.response_error(task, crate::Error::from(e))); - } + let rows = held + .finish::<_, crate::Error>(rows) + .map_err(|e| self.response_error(task, e))?; let mut matched = Vec::with_capacity(rows.len()); for (key, body) in rows { diff --git a/nodedb/src/data/executor/handlers/transaction/write_version_parity_tests.rs b/nodedb/src/data/executor/handlers/transaction/write_version_parity_tests.rs index bddce48d2..a9c361c6f 100644 --- a/nodedb/src/data/executor/handlers/transaction/write_version_parity_tests.rs +++ b/nodedb/src/data/executor/handlers/transaction/write_version_parity_tests.rs @@ -16,7 +16,6 @@ use nodedb_types::geometry::{Geometry, SridGeometry}; use nodedb_types::{DatabaseId, QualifiedCollection, RlsWriteCheck, Surrogate, Value}; use crate::bridge::envelope::{Response, Status}; -use crate::control::wal_replication::transaction_redo::collections::written_collections; use crate::control::wal_replication::transaction_redo::sum_targets::redo_sum_targets; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; @@ -24,6 +23,7 @@ use crate::data::executor::handlers::control::calvin::CalvinExecCtx; use crate::data::executor::task::ExecutionTask; use crate::types::Lsn; use crate::wal::{RedoRecord, RedoSubRecord}; +use nodedb_physical::physical_plan::written_collections; /// The LSN every install and every oracle record names. const LSN: u64 = 300; diff --git a/nodedb/src/data/executor/handlers/truncate.rs b/nodedb/src/data/executor/handlers/truncate.rs index f0a408bc1..8771d0a10 100644 --- a/nodedb/src/data/executor/handlers/truncate.rs +++ b/nodedb/src/data/executor/handlers/truncate.rs @@ -9,9 +9,8 @@ use crate::bridge::envelope::{ErrorCode, Response, WriteSetEntry}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::core_loop::fail_stop::FailStopCause; use crate::data::executor::enforcement::materialized_sum::divergence::SumTargetCheck; -use crate::data::executor::enforcement::write_hook; use crate::data::executor::handlers::partial_refusal::refusal_after_rows; -use crate::data::executor::handlers::transaction::stage_write::stored_row_identity; +use crate::data::executor::handlers::truncate_row::{TruncateProgress, TruncateScope}; use crate::data::executor::response_codec; use crate::data::executor::task::ExecutionTask; @@ -29,8 +28,10 @@ pub(in crate::data::executor) struct TruncateParams<'a> { impl CoreLoop { /// TRUNCATE: delete all documents in a collection without filter scanning. /// - /// Iterates the DOCUMENTS table prefix and deletes every key. Cascades to - /// inverted index, secondary indexes, and document cache. The + /// Removes every row's current version. A plain collection deletes each + /// row. A bitemporal collection closes each row with a tombstone version + /// and keeps its history. Cascades to the inverted index, secondary + /// indexes, and document cache. The /// collection's graph edges are cut by `TruncateEdges`. Returns /// `{"truncated": N}` payload. /// @@ -53,23 +54,13 @@ impl CoreLoop { } = params; debug!(core = self.core_id, %collection, "truncate"); - // Collect all document IDs in this collection. - let all_ids = match self.scan_matching_documents( - task.request.database_id.as_u64(), - tid, - collection, - &[], - ) { - Ok(ids) => ids, - Err(e) => { - return self.response_error(task, ErrorCode::from(e)); - } - }; - - // Gate secondary-vector maintenance once for the whole statement so a - // collection with no vector field pays nothing — mirrors - // `execute_bulk_delete`'s `has_vectors` gate. + // Every row's current version: the newest version of each row on a + // bitemporal collection, the plain table on any other. let database_id = task.request.database_id.as_u64(); + let all_ids = match self.current_row_keys(database_id, tid, collection) { + Ok(keys) => keys, + Err(e) => return self.response_error(task, ErrorCode::from(e)), + }; // Materialized-sum coverage verification (LEADER-ONLY), identical in // contract to the bulk-DML paths: the resolution was derived from a @@ -127,162 +118,42 @@ impl CoreLoop { Err(e) => return self.response_error(task, e), } - // Delete each document with full cascade. - let mut truncated = 0u64; - // One post-apply `Delete` redo entry per removed row, in removal - // order, followed by the target rows its fold rewrote. - // `wal_append_document_op` mints no pre-dispatch record for - // `DocumentOp::Truncate`, so these entries are the only record of the - // removals WAL replay and a point-in-time restore apply. - let mut write_set: Vec = Vec::new(); - // Surrogates removed so far. A refusal part-way removes their text - // from the inverted index. A full TRUNCATE empties it in one purge. - let mut removed: Vec = Vec::new(); + // A bitemporal row is closed, never erased: its history stays readable + // until the retention purge. One system time closes every row, so the + // TRUNCATE removes them at one instant, as a `DELETE` with no `WHERE`. + let bitemporal_sys_from_ms = self + .is_bitemporal(database_id, tid, collection) + .then(|| self.bitemporal_now_ms()); + let scope = TruncateScope { + task, + database_id, + tid, + collection, + resolved_sum_targets, + declared_primary_key, + strict_schema: strict_schema.as_ref(), + has_vectors, + bitemporal_sys_from_ms, + }; + // Each document leaves with full cascade, one transaction per row. + let mut progress = TruncateProgress::default(); for storage_key in &all_ids { - // One transaction per removed row, shared with the materialized-sum - // delta that row owes — identical to `execute_bulk_delete`, so a - // TRUNCATE and a `DELETE` with no predicate leave the same totals. - // A refusal after a removal committed keeps the rows removed so - // far, and carries their entries so they are journalled. - let row_txn = match self.sparse.begin_write() { - Ok(txn) => txn, - Err(e) => { - let code = refusal_after_rows(truncated, e); - return self.truncate_refusal(task, tid, collection, code, &removed, write_set); - } - }; - // A delete error refuses the TRUNCATE: read as an absent row, it - // would leave the row stored under a success. - let deleted_bytes = - match self - .sparse - .delete_in_txn(&row_txn, database_id, tid, collection, storage_key) - { - Ok(bytes) => bytes, - Err(e) => { - let code = refusal_after_rows(truncated, e); - return self - .truncate_refusal(task, tid, collection, code, &removed, write_set); - } - }; - // The row's secondary-index entries leave in the row's own - // transaction, so the row and its entries go together or not at all. - if deleted_bytes.is_some() - && let Err(e) = self.sparse.delete_indexes_for_document_in_txn( - &row_txn, - database_id, - tid, - collection, - storage_key, - ) - { - let code = refusal_after_rows(truncated, e); - return self.truncate_refusal(task, tid, collection, code, &removed, write_set); - } - let mut target_writes = Vec::new(); - if let Some(bytes) = deleted_bytes.as_deref() { - match write_hook::run( - self, - &row_txn, - &write_hook::HookCtx { - database_id, - tid, - collection, - resolved_targets: resolved_sum_targets, - deferred_sum_targets: &[], - wal_lsn: task.wal_lsn(), - }, - write_hook::WriteImages::Delete { - old: write_hook::ImageBody::Stored(bytes), - }, - ) { - // The row's BALANCED contribution was settled for the whole - // statement above, before any row was removed; taking it - // again here counts the same removal twice. - Ok(outcome) => target_writes = outcome.target_writes, - Err(e) => { - let code = refusal_after_rows(truncated, e); - return self - .truncate_refusal(task, tid, collection, code, &removed, write_set); - } - } - } - if let Err(e) = row_txn.commit() { - let code = refusal_after_rows( - truncated, - crate::Error::Storage { - engine: "sparse".into(), - detail: format!("truncate commit: {e}"), - }, - ); - return self.truncate_refusal(task, tid, collection, code, &removed, write_set); - } - if let Some(deleted_bytes) = deleted_bytes.as_deref() { - let surrogate = storage_key.surrogate(); - // The identity INSERT minted for this row, read from the - // pre-image the delete returned: the declared primary key - // when the collection declares one, else the decimal - // surrogate. The redo entry and the delete event share it. - let row_identity = stored_row_identity( - deleted_bytes, - strict_schema.as_ref(), - declared_primary_key, - *storage_key, - ); - // The removed rows' index values record no version, so the - // first removal records a wipe that bounds every index read. - if truncated == 0 { - self.note_task_collection_wipe(task, tid, collection); - } - // Record the removal's version against the row's surrogate - // and collection, as a point delete does. - self.note_surrogate_write(task, tid, collection, surrogate.as_u32()); - // The row's text leaves the inverted index with every other - // row's, in one purge once the loop ends. - removed.push(surrogate); - // Cascade: secondary HNSW vector index. The put path indexed - // this row's vectors under its surrogate; truncate must - // soft-delete those nodes and drop the reverse-map entry, or - // the leaked vector keeps scoring in KNN search in the same - // process (mirrors `execute_bulk_delete`'s vector cascade). - if has_vectors { - self.remove_document_vector_indexes(database_id, tid, collection, *storage_key); - } - write_set.push(WriteSetEntry::delete( - surrogate.as_u32(), - row_identity.clone(), - )); - // The collection's edges keep their place here: the - // TRUNCATE's transaction cuts them with one `TruncateEdges` - // per vShard, at its ordinal. - self.doc_cache.invalidate( - task.request.database_id.as_u64(), - tid, - collection, - storage_key, - ); - // Emit a delete event per removed row to the Event Plane, so - // AFTER-DELETE triggers and CDC/change-stream consumers see - // each row TRUNCATE removed — mirroring `execute_point_delete` - // and `execute_bulk_delete`'s single-row emit. `deleted_bytes` - // is the prior stored bytes `sparse.delete` returned above. - // Emitted per row rather than a single `WriteOp::BulkDelete` - // summary: that variant is aggregate metadata the Event - // Plane's WAL replay reconstructs only when the live per-row - // events were lost, and per-row events are what ROW-level - // AFTER-DELETE triggers match on (see - // `event::trigger::dispatcher::single`). - self.emit_document_delete_event( + if let Err(code) = self.truncate_row(&scope, storage_key, &mut progress) { + return self.truncate_refusal( task, tid, collection, - row_identity, - Some(deleted_bytes), + code, + &progress.removed, + progress.write_set, ); - truncated += 1; } - write_set.extend(write_hook::target_write_set(&target_writes)); } + let TruncateProgress { + truncated, + write_set, + removed: _, + } = progress; // Every row is removed: empty the collection's inverted index in one // purge. Its analyzer, language, and fuzzy configuration stay. @@ -291,6 +162,16 @@ impl CoreLoop { crate::types::TenantId::new(tid), collection, ) { + // The rows are removed, but the index still holds their text. That + // cannot be undone here, so the core fail-stops, as in + // `truncate_refusal`. Restart replay rebuilds the index. + self.fail_stop_core( + FailStopCause::PostInstallFailed, + &format!( + "TRUNCATE of '{collection}' removed {truncated} rows, then emptying its \ + inverted index failed: {e}" + ), + ); let code = refusal_after_rows(truncated, ErrorCode::from(e)); return self.refusal_with_landed_rows(task, code, write_set); } @@ -391,3 +272,140 @@ impl CoreLoop { } } } + +#[cfg(test)] +mod tests { + //! TRUNCATE on a bitemporal collection closes every current version and + //! keeps the history before it. + + use super::*; + use crate::bridge::envelope::{RowEffect, Status}; + use crate::data::executor::core_loop::tests::{make_core_with_dir, make_default_task}; + use crate::data::executor::doc_format; + use crate::engine::document::store::{CollectionConfig, IndexPath}; + use crate::engine::sparse::btree_versioned::{VersionedIndexEntry, VersionedPut}; + use crate::types::{DatabaseId, TenantId}; + use nodedb_types::{StorageKey, Surrogate}; + + const TID: u64 = 1; + const LEDGER: &str = "ledger"; + /// The system time every seeded version lands at, before the TRUNCATE. + const SEEDED_AT: i64 = 1_000; + + fn key(surrogate: u32) -> StorageKey { + StorageKey::for_surrogate(Surrogate(surrogate)) + } + + /// A bitemporal `ledger` with a secondary index on `kind`, holding rows + /// 1 (`x`), 2 (`y`) and 3 (`x`) at `SEEDED_AT`. + fn bitemporal_ledger(core: &mut CoreLoop, database_id: u64) { + let mut config = CollectionConfig::new(LEDGER); + config.bitemporal = true; + config.index_paths.push(IndexPath::new("kind")); + core.doc_configs.insert( + ( + DatabaseId::new(database_id), + TenantId::new(TID), + LEDGER.to_string(), + ), + config, + ); + for (surrogate, kind) in [(1, "x"), (2, "y"), (3, "x")] { + let body = doc_format::encode_to_msgpack(&serde_json::json!({ "kind": kind })); + core.sparse + .versioned_put(VersionedPut { + database_id, + tenant: TID, + coll: LEDGER, + doc_id: &key(surrogate), + sys_from_ms: SEEDED_AT, + valid_from_ms: i64::MIN, + valid_until_ms: i64::MAX, + body: &body, + }) + .expect("seed version"); + core.sparse + .versioned_index_put(VersionedIndexEntry { + database_id, + tenant: TID, + coll: LEDGER, + field: "kind", + value: kind, + doc_id: &key(surrogate), + sys_from_ms: SEEDED_AT, + }) + .expect("seed index entry"); + } + } + + fn version_at( + core: &CoreLoop, + database_id: u64, + surrogate: u32, + sys_cutoff_ms: Option, + ) -> Option> { + core.sparse + .versioned_get_as_of( + database_id, + TID, + LEDGER, + &key(surrogate), + sys_cutoff_ms, + None, + ) + .expect("read version") + } + + #[test] + fn truncate_closes_every_current_version_of_a_bitemporal_collection() { + let dir = tempfile::tempdir().expect("tempdir"); + let (mut core, _req, _resp) = make_core_with_dir(dir.path()); + let task = make_default_task(); + let database_id = task.request.database_id.as_u64(); + bitemporal_ledger(&mut core, database_id); + + let response = core.execute_truncate( + &task, + TID, + TruncateParams { + collection: LEDGER, + resolved_sum_targets: &[], + declared_primary_key: None, + }, + ); + + assert_eq!(response.status, Status::Ok, "{:?}", response.error_code); + let payload = + nodedb_types::json_from_msgpack(response.payload.as_bytes()).expect("payload decodes"); + assert_eq!(payload["truncated"], serde_json::json!(3)); + for surrogate in [1, 2, 3] { + assert!( + version_at(&core, database_id, surrogate, None).is_none(), + "row {surrogate} has no current version" + ); + assert!( + version_at(&core, database_id, surrogate, Some(SEEDED_AT)).is_some(), + "row {surrogate} keeps its history" + ); + } + let current_x = core + .sparse + .versioned_index_lookup_as_of(database_id, TID, LEDGER, "kind", "x", None) + .expect("index lookup"); + assert!(current_x.is_empty(), "the index tombstones every value"); + + // Each removal journals its tombstone at one statement system time. + let stamps: Vec> = response + .write_set + .iter() + .filter_map(|entry| match entry.effect { + RowEffect::Delete { system_from_ms } => Some(system_from_ms), + _ => None, + }) + .collect(); + assert_eq!(stamps.len(), 3); + let first = stamps[0].expect("a versioned tombstone"); + assert!(first > SEEDED_AT); + assert!(stamps.iter().all(|stamp| *stamp == Some(first))); + } +} diff --git a/nodedb/src/data/executor/handlers/truncate_row.rs b/nodedb/src/data/executor/handlers/truncate_row.rs new file mode 100644 index 000000000..60e4f0d35 --- /dev/null +++ b/nodedb/src/data/executor/handlers/truncate_row.rs @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Removing one row of a TRUNCATE: its current version, its secondary-index +//! entries and the materialized-sum deltas it owes, in one transaction, then +//! its journal entry, its delete event and its in-memory cascades. + +use nodedb_physical::physical_plan::ResolvedSumTarget; +use nodedb_types::columnar::StrictSchema; +use nodedb_types::{StorageKey, Surrogate}; + +use crate::bridge::envelope::{ErrorCode, WriteSetEntry}; +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::core_loop::redo_image::removed_row_image; +use crate::data::executor::enforcement::write_hook; +use crate::data::executor::handlers::partial_refusal::refusal_after_rows; +use crate::data::executor::handlers::point::delete_version::CurrentRowRemoval; +use crate::data::executor::handlers::transaction::stage_write::stored_row_identity; +use crate::data::executor::task::ExecutionTask; + +/// What every row of one TRUNCATE is removed under. +pub(in crate::data::executor) struct TruncateScope<'a> { + pub task: &'a ExecutionTask, + pub database_id: u64, + pub tid: u64, + pub collection: &'a str, + pub resolved_sum_targets: &'a [ResolvedSumTarget], + /// The collection's declared `PRIMARY KEY` column, when it has one. + pub declared_primary_key: Option<&'a str>, + /// `Some` for a strict collection, whose pre-images are Binary Tuples. + pub strict_schema: Option<&'a StrictSchema>, + pub has_vectors: bool, + /// The tombstones' system time on a bitemporal collection, `None` on any + /// other. + pub bitemporal_sys_from_ms: Option, +} + +/// What a TRUNCATE has removed so far. +#[derive(Default)] +pub(in crate::data::executor) struct TruncateProgress { + /// Rows removed. + pub truncated: u64, + /// One `Delete` redo entry per removed row, in removal order, each + /// followed by the target rows its fold rewrote. The plan mints no + /// pre-dispatch record, so these entries are the removals' only record. + pub write_set: Vec, + /// Surrogates of the removed rows. A refusal part-way removes their text + /// from the inverted index. + pub removed: Vec, +} + +impl CoreLoop { + /// Remove one row and account it into `progress`. + /// + /// The row's removal, its secondary-index entries and the + /// materialized-sum deltas it owes share one transaction, as in + /// `execute_bulk_delete`. `Err` is the refusal code. The rows removed + /// before it stay removed and journalled. + pub(in crate::data::executor) fn truncate_row( + &mut self, + s: &TruncateScope<'_>, + storage_key: &StorageKey, + progress: &mut TruncateProgress, + ) -> Result<(), ErrorCode> { + let refuse = |truncated: u64, e: crate::Error| refusal_after_rows(truncated, e); + let (database_id, tid, collection) = (s.database_id, s.tid, s.collection); + let row_txn = self + .sparse + .begin_write() + .map_err(|e| refuse(progress.truncated, e))?; + // A delete error refuses the TRUNCATE: read as an absent row, it + // leaves the row stored under a success. + let removal = self + .remove_current_row_in_txn( + &row_txn, + CurrentRowRemoval { + database_id, + tid, + collection, + storage_key, + bitemporal_sys_from_ms: s.bitemporal_sys_from_ms, + }, + ) + .map_err(|e| refuse(progress.truncated, e))?; + let Some(deleted_bytes) = removal.prior else { + return Ok(()); + }; + self.sparse + .delete_indexes_for_document_in_txn(&row_txn, database_id, tid, collection, storage_key) + .map_err(|e| refuse(progress.truncated, e))?; + // The row's BALANCED contribution was settled for the whole statement + // before any row was removed; taking it again here counts it twice. + let outcome = write_hook::run( + self, + &row_txn, + &write_hook::HookCtx { + database_id, + tid, + collection, + resolved_targets: s.resolved_sum_targets, + deferred_sum_targets: &[], + wal_lsn: s.task.wal_lsn(), + }, + write_hook::WriteImages::Delete { + old: write_hook::ImageBody::Stored(&deleted_bytes), + }, + ) + .map_err(|e| refusal_after_rows(progress.truncated, e))?; + row_txn.commit().map_err(|e| { + refuse( + progress.truncated, + crate::Error::Storage { + engine: "sparse".into(), + detail: format!("truncate commit: {e}"), + }, + ) + })?; + + let surrogate = storage_key.surrogate(); + // The identity INSERT minted for this row: the declared primary key + // when the collection declares one, else the decimal surrogate. The + // redo entry and the delete event share it. + let row_identity = stored_row_identity( + &deleted_bytes, + s.strict_schema, + s.declared_primary_key, + *storage_key, + ); + // The removed rows' index values record no version, so the first + // removal records a wipe that bounds every index read. + if progress.truncated == 0 { + self.note_task_collection_wipe(s.task, tid, collection); + } + self.note_surrogate_write(s.task, tid, collection, surrogate.as_u32()); + // The row's text leaves the inverted index with every other row's, + // in one purge once the statement's rows are removed. + progress.removed.push(surrogate); + // The put path indexed this row's vectors under its surrogate. A + // leaked vector keeps scoring in KNN search in the same process. + if s.has_vectors { + self.remove_document_vector_indexes(database_id, tid, collection, *storage_key); + } + // A bitemporal tombstone carries its version key, so replay closes + // the same version. + progress.write_set.push(removed_row_image( + surrogate.as_u32(), + row_identity.clone(), + removal.tombstone_sys_from_ms, + )); + // The collection's edges keep their place here: the TRUNCATE's + // transaction cuts them with one `TruncateEdges` per vShard. + self.doc_cache + .invalidate(database_id, tid, collection, storage_key); + // One delete event per removed row, as point and bulk delete emit: + // ROW-level AFTER-DELETE triggers match on per-row events. + self.emit_document_delete_event( + s.task, + tid, + collection, + row_identity, + Some(deleted_bytes.as_slice()), + ); + progress.truncated += 1; + progress + .write_set + .extend(write_hook::target_write_set(&outcome.target_writes)); + Ok(()) + } +} diff --git a/nodedb/src/data/executor/handlers/update_from_join_collect.rs b/nodedb/src/data/executor/handlers/update_from_join_collect.rs index fa5e080a5..2afa1e53e 100644 --- a/nodedb/src/data/executor/handlers/update_from_join_collect.rs +++ b/nodedb/src/data/executor/handlers/update_from_join_collect.rs @@ -11,19 +11,14 @@ //! diverge on which rows match or what post-image each carries (mirrors how //! `collect_merge_plan` is shared between the MERGE resolve and apply passes). -use redb::{ReadableDatabase, ReadableTable}; - -use nodedb_types::StorageKey; use nodedb_types::columnar::StrictSchema; use crate::bridge::scan_filter::ScanFilter; use crate::data::executor::core_loop::CoreLoop; -use crate::data::executor::core_loop::filter_match::matches_with_resolved_schema; use crate::data::executor::doc_format; use crate::data::executor::handlers::join_source_map::JoinSource; use crate::data::executor::task::ExecutionTask; -use crate::engine::sparse::btree::{KeyedTable, invalid_storage_key_err}; -use crate::types::{DatabaseId, TenantId, TxnId}; +use crate::types::{DatabaseId, TenantId}; use nodedb_physical::physical_plan::UpdateValue; use super::update_from_join::ResolvedUpdateRow; @@ -50,20 +45,6 @@ pub(in crate::data::executor) struct CollectUpdateRows<'a> { pub bitemporal_sys_from_ms: Option, } -/// Borrowed inputs for [`CoreLoop::scan_target_rows`], bundled to keep the -/// overlay-aware target scan within argument limits. -struct ScanTargetRows<'a> { - database_id: u64, - tid: u64, - target_collection: &'a str, - target_filters: &'a [ScanFilter], - strict_schema: Option<&'a StrictSchema>, - txn_id: Option, - target_coll_key: &'a (DatabaseId, TenantId, String), - /// Read the current versions of a bitemporal target. - bitemporal: bool, -} - impl CoreLoop { /// Resolve every target row matched by the join into its post-image without /// writing. Shared by the write path and the RESOLVE pass. @@ -101,16 +82,13 @@ impl CoreLoop { // each matched row's CURRENT body (overlay put superseding base, staged // insert appended), so the body is used directly — a base `sparse.get` // would miss a row this transaction only staged. - let target_rows = self.scan_target_rows(ScanTargetRows { - database_id, - tid, - target_collection, - target_filters, - strict_schema, - txn_id, - target_coll_key: &target_coll_key, - bitemporal: bitemporal_sys_from_ms.is_some(), - })?; + // Each row is at its current version, so a bitemporal target matches. + let target_rows = match txn_id { + Some(txn_id) => self.txn_matching_rows(txn_id, &target_coll_key, target_filters)?, + None => { + self.current_matching_rows(database_id, tid, target_collection, target_filters)? + } + }; let mut rows: Vec = Vec::new(); for (key, current_bytes) in target_rows { @@ -271,170 +249,4 @@ impl CoreLoop { } Ok(rows) } - - /// Range-scan the target collection, returning each row that passes every - /// target-only filter as `(doc_id, current_stored_body)` — decoding strict - /// Binary Tuples to JSON for filter evaluation when the target is - /// strict-mode. The body is the row's CURRENT stored form (strict Binary - /// Tuple or MessagePack), returned so the caller need not re-fetch it. - /// - /// When `txn_id` is `Some`, the transaction's staging overlay is folded over - /// the base result: a staged tombstone hides its base row, a staged put - /// replaces the base body (re-checked against the SAME target filters via the - /// strict-aware matcher), and a staged put absent from base is appended when - /// it passes the filters. `None` (autocommit) returns the base-filtered rows - /// unchanged — byte-identical to the pre-staging behavior. - fn scan_target_rows( - &self, - args: ScanTargetRows<'_>, - ) -> crate::Result)>> { - let ScanTargetRows { - database_id, - tid, - target_collection, - target_filters, - strict_schema, - txn_id, - target_coll_key, - bitemporal, - } = args; - let identity_column = self.identity_column(database_id, tid, target_collection); - let mut rows = if bitemporal { - self.scan_current_versions(database_id, tid, target_collection, |key, body| { - matches_with_resolved_schema( - strict_schema, - target_filters, - key, - body, - &identity_column, - ) - })? - } else { - self.scan_plain_rows(database_id, tid, target_collection, |key, body| { - matches_with_resolved_schema( - strict_schema, - target_filters, - key, - body, - &identity_column, - ) - })? - }; - - // Read-your-own-writes: fold the transaction's staging overlay over the - // base-filtered rows. The overlay's staged bodies are the same canonical - // stored form as base bodies, so the strict-aware matcher re-checks a - // staged put against the same target filters — a staged insert/update - // that satisfies the predicate is surfaced, one that no longer does is - // dropped, exactly as for a base row. - if let Some(txn_id) = txn_id { - // `merge_overlay_into_scan` takes an infallible - // `Fn(&StorageKey, &[u8]) -> bool` predicate, so a division/modulo- - // by-zero is captured via this `Cell` side-channel and checked once - // the merge returns. - let raw_matches = - self.strict_aware_matcher(database_id, tid, target_collection, target_filters); - let predicate_err: std::cell::Cell> = - std::cell::Cell::new(None); - let matches = |row_key: &StorageKey, body: &[u8]| match raw_matches(row_key, body) { - Ok(b) => b, - Err(e) => { - predicate_err.set(Some(e)); - false - } - }; - self.merge_overlay_into_scan(txn_id, target_coll_key, &mut rows, &matches); - if let Some(e) = predicate_err.take() { - return Err(crate::Error::from(e)); - } - } - Ok(rows) - } - - /// Every current version of a bitemporal collection whose body passes - /// `matches`, as `(doc_id, stored_body)`. - fn scan_current_versions( - &self, - database_id: u64, - tid: u64, - collection: &str, - matches: impl Fn(&StorageKey, &[u8]) -> Result, - ) -> crate::Result)>> { - // The scan's predicate is infallible, so an evaluation error is kept - // here and returned once the scan ends. - let predicate_err: std::cell::Cell> = - std::cell::Cell::new(None); - let predicate = |key: &StorageKey, body: &[u8]| match matches(key, body) { - Ok(matched) => matched, - Err(e) => { - predicate_err.set(Some(e)); - false - } - }; - let rows = self.sparse.versioned_scan_as_of( - crate::engine::sparse::btree_versioned::VersionedScanParams { - database_id, - tenant: tid, - coll: collection, - sys_cutoff_ms: None, - valid_at_ms: None, - limit: usize::MAX, - }, - &predicate, - &crate::engine::sparse::scan_stop::never_stop, - )?; - if let Some(e) = predicate_err.take() { - return Err(crate::Error::from(e)); - } - Ok(rows) - } - - /// Every row of a collection whose body passes `matches`, as - /// `(doc_id, stored_body)`. - fn scan_plain_rows( - &self, - database_id: u64, - tid: u64, - target_collection: &str, - matches: impl Fn(&StorageKey, &[u8]) -> Result, - ) -> crate::Result)>> { - let prefix = crate::engine::sparse::btree::coll_prefix(database_id, tid, target_collection); - let end = format!("{prefix}\u{ffff}"); - - let read_txn = self - .sparse - .db() - .begin_read() - .map_err(|e| crate::Error::Storage { - engine: "sparse".into(), - detail: format!("read txn: {e}"), - })?; - let table = read_txn - .open_table(crate::engine::sparse::btree::DOCUMENTS) - .map_err(|e| crate::Error::Storage { - engine: "sparse".into(), - detail: format!("open table: {e}"), - })?; - - let mut rows: Vec<(StorageKey, Vec)> = Vec::new(); - if let Ok(range) = table.range(prefix.as_str()..end.as_str()) { - for entry in range.flatten() { - let full_key = entry.0.value(); - let value_bytes = entry.1.value(); - let Some(rest) = full_key.strip_prefix(&prefix) else { - continue; - }; - let key = StorageKey::parse(rest).ok_or_else(|| { - invalid_storage_key_err(KeyedTable::Documents, target_collection, rest) - })?; - // Goes through the same primitive the overlay fold uses, so a - // schemaless row with no `id` field matches `WHERE id ...` - // here exactly as it does once staged. - if matches(&key, value_bytes).map_err(crate::Error::from)? { - rows.push((key, value_bytes.to_vec())); - } - } - } - Ok(rows) - } } diff --git a/nodedb/src/data/executor/handlers/update_from_join_write.rs b/nodedb/src/data/executor/handlers/update_from_join_write.rs index a32bce5db..b5964cf8a 100644 --- a/nodedb/src/data/executor/handlers/update_from_join_write.rs +++ b/nodedb/src/data/executor/handlers/update_from_join_write.rs @@ -101,17 +101,14 @@ impl CoreLoop { { for row in &rows { for image in [&row.old_body, &row.body] { - if let Err(e) = - crate::data::executor::enforcement::period_lock::check_period_lock( - &self.sparse, - database_id, - tid, - target_collection, - image, - lock, - resolved_sum_targets, - ) - { + if let Err(e) = self.check_period_lock_at( + database_id, + tid, + target_collection, + image, + lock, + resolved_sum_targets, + ) { return Err(self.response_error(task, e)); } } diff --git a/nodedb/src/data/executor/handlers/upsert/exec/batch.rs b/nodedb/src/data/executor/handlers/upsert/exec/batch.rs index 2220971b2..04a2849dd 100644 --- a/nodedb/src/data/executor/handlers/upsert/exec/batch.rs +++ b/nodedb/src/data/executor/handlers/upsert/exec/batch.rs @@ -147,7 +147,6 @@ impl CoreLoop { database_id, tid, collection, - bitemporal: self.is_bitemporal(database_id, tid, collection), strict_schema: strict_schema.as_ref(), identity_column: &identity_column, }; diff --git a/nodedb/src/data/executor/handlers/upsert/exec/batch_plan.rs b/nodedb/src/data/executor/handlers/upsert/exec/batch_plan.rs index c30bd5dc0..968b36ff1 100644 --- a/nodedb/src/data/executor/handlers/upsert/exec/batch_plan.rs +++ b/nodedb/src/data/executor/handlers/upsert/exec/batch_plan.rs @@ -37,7 +37,6 @@ pub(super) struct PageScope<'a> { pub database_id: u64, pub tid: u64, pub collection: &'a str, - pub bitemporal: bool, pub strict_schema: Option<&'a StrictSchema>, pub identity_column: &'a str, } @@ -90,22 +89,14 @@ impl CoreLoop { } None => { let storage_key = StorageKey::for_surrogate(surrogate); - let committed = if scope.bitemporal { - self.sparse.versioned_get_current( + let committed = self + .current_row_body( scope.database_id, scope.tid, scope.collection, &storage_key, ) - } else { - self.sparse.get( - scope.database_id, - scope.tid, - scope.collection, - &storage_key, - ) - } - .map_err(ErrorCode::from)?; + .map_err(ErrorCode::from)?; let (post, assigned) = match &committed { None => (value.clone(), None), Some(current) => { diff --git a/nodedb/src/data/executor/handlers/upsert/exec/dispatch.rs b/nodedb/src/data/executor/handlers/upsert/exec/dispatch.rs index da5b83053..7a1cc3316 100644 --- a/nodedb/src/data/executor/handlers/upsert/exec/dispatch.rs +++ b/nodedb/src/data/executor/handlers/upsert/exec/dispatch.rs @@ -106,23 +106,14 @@ impl CoreLoop { } }); - // Check if document already exists. Bitemporal collections consult - // the versioned table's current-state view (reverse-scan to newest - // non-tombstone); non-bitemporal collections use the legacy point - // lookup. - let bitemporal = self.is_bitemporal(database_id, tid, collection); // Computed once for the whole statement: the schemaless half of this // check is an unindexed `vector_params` scan, so it must not be paid // per branch. Gates the overwrite branch's removal of the prior HNSW // node; a non-vector collection pays nothing. let has_vectors = self.collection_has_vectors(database_id, tid, collection); let key = nodedb_types::StorageKey::for_surrogate(surrogate); - let existing = if bitemporal { - self.sparse - .versioned_get_current(database_id, tid, collection, &key) - } else { - self.sparse.get(database_id, tid, collection, &key) - }; + // Whether the row already exists, at its current version. + let existing = self.current_row_body(database_id, tid, collection, &key); match existing { Ok(Some(current_bytes)) => self.execute_upsert_overwrite( diff --git a/nodedb/src/data/executor/handlers/vector_search_exec.rs b/nodedb/src/data/executor/handlers/vector_search_exec.rs index e66636a7e..b59155d67 100644 --- a/nodedb/src/data/executor/handlers/vector_search_exec.rs +++ b/nodedb/src/data/executor/handlers/vector_search_exec.rs @@ -45,7 +45,8 @@ impl CoreLoop { let Some(key) = hit.id.storage_key() else { return Ok(hit); }; - if let Some(bytes) = self.sparse.get(database_id, tid, collection, &key)? { + // The row at its current version, so a bitemporal collection attaches. + if let Some(bytes) = self.current_row_body(database_id, tid, collection, &key)? { let format = self.sparse_body_format( crate::types::DatabaseId::new(database_id), crate::types::TenantId::new(tid), diff --git a/nodedb/src/data/executor/handlers/vector_write.rs b/nodedb/src/data/executor/handlers/vector_write.rs index 8abe344b2..907f5cd8b 100644 --- a/nodedb/src/data/executor/handlers/vector_write.rs +++ b/nodedb/src/data/executor/handlers/vector_write.rs @@ -113,16 +113,8 @@ impl CoreLoop { // Capture the surrogate before deletion so we can fetch the // payload row from the sparse store and update the bitmap. The // bitmap stores node-id -> field-value membership; without the - // original field values we cannot remove the entries cleanly. - // - // Asymmetric with the insert path (`vector_upsert`): insert - // atomically rolls back the HNSW node if the sparse write fails, - // because a phantom node would be returned by future searches. - // Delete is best-effort cleanup — if the sparse read or decode - // fails we still drop the HNSW node and skip bitmap cleanup. - // Phantom bitmap entries are safe (the bitmap is filtered against - // live node ids on read), so leaving them is preferable to - // aborting the delete and leaking the vector. + // original field values we cannot remove the entries cleanly, so + // a row that does not read or decode refuses the delete. let surrogate_opt = self .vector_collections .get(&index_key) @@ -130,14 +122,35 @@ impl CoreLoop { if let Some(surrogate) = surrogate_opt { let row_key = nodedb_types::StorageKey::for_surrogate(surrogate); - let fields = - match self - .sparse - .get(task.request.database_id.as_u64(), tid, collection, &row_key) - { - Ok(Some(bytes)) => decode_payload_lowercased(&bytes).ok(), - _ => None, - }; + // The row at its current version, so a bitemporal collection's + // payload entries leave too. A read or decode error refuses the + // delete before anything is removed. A row that is gone has no + // payload entries to remove. + let fields = match self.current_row_body( + task.request.database_id.as_u64(), + tid, + collection, + &row_key, + ) { + Ok(Some(bytes)) => match decode_payload_lowercased(&bytes) { + Ok(fields) => Some(fields), + Err(e) => { + let detail = e.to_string(); + crate::diag::vector_payload_undecodable(collection, &detail); + return self.response_error( + task, + ErrorCode::Internal { + detail: format!( + "vector delete: payload of the row behind vector \ + {vector_id} in '{collection}' does not decode: {detail}" + ), + }, + ); + } + }, + Ok(None) => None, + Err(e) => return self.response_error(task, ErrorCode::from(e)), + }; if let Some(fields) = fields && let Some(coll) = self.vector_collections.get_mut(&index_key) { diff --git a/nodedb/src/data/executor/handlers/verify_hash_chain.rs b/nodedb/src/data/executor/handlers/verify_hash_chain.rs index 3910d2376..52e9d2a23 100644 --- a/nodedb/src/data/executor/handlers/verify_hash_chain.rs +++ b/nodedb/src/data/executor/handlers/verify_hash_chain.rs @@ -93,30 +93,8 @@ impl CoreLoop { visit(&key.to_string(), &view); Ok(()) }; - if self.is_bitemporal(database_id, tid, collection) { - let rows = self.sparse.versioned_scan_as_of( - crate::engine::sparse::btree_versioned::VersionedScanParams { - database_id, - tenant: tid, - coll: collection, - sys_cutoff_ms: None, - valid_at_ms: None, - limit: usize::MAX, - }, - &|_: &StorageKey, _: &[u8]| true, - &crate::engine::sparse::scan_stop::never_stop, - )?; - for (key, bytes) in &rows { - decode_and_visit(key, bytes.as_slice())?; - } - return Ok(()); - } - self.sparse.scan_documents_for_each( - database_id, - tid, - collection, - usize::MAX, - |key, bytes| decode_and_visit(key, bytes), - ) + self.for_each_current_row(database_id, tid, collection, |key, bytes| { + decode_and_visit(key, bytes).map(|()| std::ops::ControlFlow::Continue(())) + }) } } diff --git a/nodedb/src/data/executor/response_codec/graph_rag.rs b/nodedb/src/data/executor/response_codec/graph_rag.rs new file mode 100644 index 000000000..4bac5db0f --- /dev/null +++ b/nodedb/src/data/executor/response_codec/graph_rag.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The GraphRAG fusion response, shared by the Data Plane, the cluster +//! coordinator and the Control-Plane response translator. +//! +//! `GraphRagResult::surrogate` is set on a row named by no graph node. The +//! response translator names that row from the catalog and clears the field. +//! The encoder omits a cleared field, so a client never sees it. + +use serde::Serialize; + +/// One fused row. +#[derive(Debug, Clone, PartialEq, Serialize, zerompk::FromMessagePack)] +#[msgpack(map)] +pub(crate) struct GraphRagResult { + pub node_id: String, + /// The row's surrogate while `node_id` is its decimal stand-in. + #[serde(skip_serializing_if = "Option::is_none")] + #[msgpack(default)] + pub surrogate: Option, + pub rrf_score: f64, + #[serde(skip_serializing_if = "Option::is_none")] + pub vector_rank: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub vector_distance: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub hop_distance: Option, +} + +/// Writes `surrogate` only while it is set. Every other optional field is +/// written as nil when absent. +impl zerompk::ToMessagePack for GraphRagResult { + fn write(&self, writer: &mut W) -> zerompk::Result<()> { + writer.write_map_len(5 + usize::from(self.surrogate.is_some()))?; + writer.write_string("node_id")?; + writer.write_string(&self.node_id)?; + if let Some(surrogate) = self.surrogate { + writer.write_string("surrogate")?; + writer.write_u32(surrogate)?; + } + writer.write_string("rrf_score")?; + writer.write_f64(self.rrf_score)?; + writer.write_string("vector_rank")?; + zerompk::ToMessagePack::write(&self.vector_rank, writer)?; + writer.write_string("vector_distance")?; + zerompk::ToMessagePack::write(&self.vector_distance, writer)?; + writer.write_string("hop_distance")?; + zerompk::ToMessagePack::write(&self.hop_distance, writer) + } +} + +/// A fusion's rows and the run's metadata. +#[derive(Debug, Serialize, zerompk::ToMessagePack, zerompk::FromMessagePack)] +#[msgpack(map)] +pub(crate) struct GraphRagResponse { + pub results: Vec, + pub metadata: GraphRagMetadata, +} + +/// What a fusion run read and reached. +#[derive(Debug, Serialize, zerompk::ToMessagePack, zerompk::FromMessagePack)] +#[msgpack(map)] +pub(crate) struct GraphRagMetadata { + pub vector_candidates: usize, + pub graph_expanded: usize, + pub truncated: bool, + /// Expanded nodes that carry no surrogate and therefore cannot intersect + /// another engine's candidates. They are traversed and returned like any + /// other node. A non-zero count beside few results means missing identity + /// bindings, not a small graph. + pub graph_unaddressable: usize, + /// Snapshot watermark LSN at the time of query execution. + pub watermark_lsn: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn row(surrogate: Option) -> GraphRagResult { + GraphRagResult { + node_id: "a".into(), + surrogate, + rrf_score: 0.5, + vector_rank: Some(0), + vector_distance: None, + hop_distance: Some(1), + } + } + + #[test] + fn named_row_omits_surrogate_and_round_trips() { + let named = row(None); + let bytes = zerompk::to_msgpack_vec(&named).expect("encode"); + let json = nodedb_types::json_from_msgpack(&bytes).expect("json"); + assert!(json.get("surrogate").is_none(), "got {json}"); + assert!(json.get("vector_distance").is_some(), "got {json}"); + let back: GraphRagResult = zerompk::from_msgpack(&bytes).expect("decode"); + assert_eq!(back, named); + } + + #[test] + fn unnamed_row_carries_its_surrogate_across_the_wire() { + let unnamed = row(Some(7)); + let bytes = zerompk::to_msgpack_vec(&unnamed).expect("encode"); + let back: GraphRagResult = zerompk::from_msgpack(&bytes).expect("decode"); + assert_eq!(back, unnamed); + } +} diff --git a/nodedb/src/data/executor/response_codec/hits.rs b/nodedb/src/data/executor/response_codec/hits.rs index ed67c75fc..c7e0706e1 100644 --- a/nodedb/src/data/executor/response_codec/hits.rs +++ b/nodedb/src/data/executor/response_codec/hits.rs @@ -88,19 +88,6 @@ pub(in crate::data::executor) struct SubgraphEdge<'a> { pub dst: &'a str, } -#[derive(Serialize, zerompk::ToMessagePack)] -#[msgpack(map)] -pub(crate) struct GraphRagResult { - pub node_id: String, - pub rrf_score: f64, - #[serde(skip_serializing_if = "Option::is_none")] - pub vector_rank: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub vector_distance: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub hop_distance: Option, -} - /// Structured response for DML with RETURNING clause. /// /// Carries one entry per affected row, with the projected column values. @@ -229,30 +216,6 @@ impl<'a> zerompk::ToMessagePack for HybridSearchHit<'a> { } } -#[derive(Serialize, zerompk::ToMessagePack)] -#[msgpack(map)] -pub(crate) struct GraphRagResponse { - pub results: Vec, - pub metadata: GraphRagMetadata, -} - -#[derive(Serialize, zerompk::ToMessagePack)] -#[msgpack(map)] -pub(crate) struct GraphRagMetadata { - pub vector_candidates: usize, - pub graph_expanded: usize, - pub truncated: bool, - /// Expanded nodes that carry no surrogate and therefore cannot intersect - /// another engine's candidates. They are traversed and returned like any - /// other node; what this reports is that the *cross-engine* view of the - /// expansion is narrower than the expansion itself. Non-zero alongside a - /// low result count means missing identity bindings, not a small graph. - pub graph_unaddressable: usize, - /// Snapshot watermark LSN at the time of query execution. - /// Consumers can use this to verify they are reading a consistent view. - pub watermark_lsn: u64, -} - #[cfg(test)] mod tests { use super::super::encode::{decode_payload_to_json, encode}; diff --git a/nodedb/src/data/executor/response_codec/mod.rs b/nodedb/src/data/executor/response_codec/mod.rs index 406271342..9cc54b52e 100644 --- a/nodedb/src/data/executor/response_codec/mod.rs +++ b/nodedb/src/data/executor/response_codec/mod.rs @@ -26,6 +26,7 @@ mod arrow; mod decode; mod encode; +mod graph_rag; mod hits; mod raw; @@ -36,11 +37,12 @@ pub(in crate::data::executor) use encode::{ encode, encode_json_as_msgpack, encode_json_vec_as_msgpack, encode_serde, encode_value_vec, }; pub(crate) use encode::{encode_affected, encode_affected_with_op, encode_count}; +pub(crate) use graph_rag::{GraphRagMetadata, GraphRagResponse, GraphRagResult}; #[allow(unused_imports)] pub(crate) use hits::ArrayAggregateResponse; pub(crate) use hits::{ - ArraySliceResponse, GraphRagMetadata, GraphRagResponse, GraphRagResult, IngestRejection, - RejectingRowsPayload, ReturningRowsReply, RowsPayload, StagedReturningReply, + ArraySliceResponse, IngestRejection, RejectingRowsPayload, ReturningRowsReply, RowsPayload, + StagedReturningReply, }; pub(in crate::data::executor) use hits::{ DocumentRow, HybridSearchHit, NeighborEntry, NeighborMultiEntry, SubgraphEdge, VectorSearchHit, @@ -48,5 +50,5 @@ pub(in crate::data::executor) use hits::{ pub(crate) use raw::{decode_raw_scan_to_docs, encode_raw_document_rows}; pub use raw::{ encode_binary_rows, flatten_hybrid_hits_to_relational_rows, flatten_to_relational_rows, - flatten_vector_hits_to_relational_rows, + flatten_vector_hits_to_relational_rows, hybrid_hit_surrogates, vector_hit_surrogates, }; diff --git a/nodedb/src/data/executor/response_codec/raw.rs b/nodedb/src/data/executor/response_codec/raw.rs index 253c6f526..0ab001de2 100644 --- a/nodedb/src/data/executor/response_codec/raw.rs +++ b/nodedb/src/data/executor/response_codec/raw.rs @@ -155,69 +155,145 @@ pub fn flatten_to_relational_rows(bytes: &[u8]) -> Vec { /// `value_to_msgpack`); the derived `zerompk` `Value` codec is tagged and would /// corrupt the row. RLS is already enforced by the Data Plane before the gather, /// so these hits are post-RLS. +/// +/// A hit whose stored body does not decode fails the call with a +/// serialization error: corruption is reported, never named from a fallback. pub fn flatten_vector_hits_to_relational_rows( bytes: &[u8], resolve_pk: impl Fn(u32) -> Option, -) -> Vec { - use crate::data::executor::handlers::hybrid_key::HybridFusionKey; +) -> crate::Result> { use nodedb_types::Value; - #[derive(zerompk::FromMessagePack)] - #[msgpack(map)] - struct Hit { - id: HybridFusionKey, - distance: f32, - doc_id: Option, - body: Option>, - } - - let hits: Vec = match zerompk::from_msgpack(bytes) { + let hits: Vec = match zerompk::from_msgpack(bytes) { Ok(h) => h, // Not a hit array (already flat, or a non-row payload): leave as-is. - Err(_) => return bytes.to_vec(), + Err(_) => return Ok(bytes.to_vec()), }; - let rows: Vec> = hits - .into_iter() - .filter_map(|h| { - let mut fields: std::collections::HashMap = match h - .body - .as_deref() - .and_then(|b| nodedb_types::value_from_msgpack(b).ok()) - { - Some(Value::Object(map)) => map, - _ => std::collections::HashMap::new(), - }; - fields - .entry("distance".to_string()) - .or_insert(Value::Float(h.distance as f64)); - // `None` for a headless hit (no surrogate binding) — never - // resolvable to a user PK. - let surrogate = h.id.storage_key().map(|k| k.surrogate().as_u32()); - let raw_identity = match surrogate { - Some(s) => Value::Integer(s as i64), - None => Value::String(h.id.to_string()), + let mut rows: Vec> = Vec::with_capacity(hits.len()); + for h in hits { + // A stored body that does not decode is corruption: the hit is + // refused, never named from a fallback. + let decoded = match h.body.as_deref() { + Some(body) => Some(nodedb_types::value_from_msgpack(body).map_err(|e| { + crate::Error::Serialization { + format: "msgpack".into(), + detail: format!( + "vector hit {} carries a stored body that does not decode: {e}", + h.id + ), + } + })?), + None => None, + }; + let mut fields: std::collections::HashMap = match decoded { + Some(Value::Object(map)) => map, + _ => std::collections::HashMap::new(), + }; + fields + .entry("distance".to_string()) + .or_insert(Value::Float(h.distance as f64)); + // `None` for a headless hit (no surrogate binding) — never + // resolvable to a user PK. + let surrogate = h.id.storage_key().map(|k| k.surrogate().as_u32()); + let raw_identity = match surrogate { + Some(s) => Value::Integer(s as i64), + None => Value::String(h.id.to_string()), + }; + if !fields.contains_key("id") { + // Prefer a catalog-resolved PK, then the DP-set doc_id, then the + // raw surrogate (or the headless sentinel) as a last resort. The + // resolver is asked exactly when `vector_hit_surrogates` lists + // the hit: both decide through `body_needs_resolved_key`. + let resolved = if body_needs_resolved_key(h.body.as_deref()) { + surrogate.and_then(&resolve_pk) + } else { + None }; - if !fields.contains_key("id") { - // Prefer a catalog-resolved PK, then the DP-set doc_id, then the - // raw surrogate (or the headless sentinel) as a last resort. - match surrogate.and_then(&resolve_pk).or(h.doc_id) { - Some(pk) => { - fields.insert("id".to_string(), Value::String(pk)); - } - None => { - fields.insert("id".to_string(), raw_identity.clone()); - } + match resolved.or(h.doc_id) { + Some(pk) => { + fields.insert("id".to_string(), Value::String(pk)); + } + None => { + fields.insert("id".to_string(), raw_identity.clone()); } } - fields - .entry("_surrogate".to_string()) - .or_insert(raw_identity); - nodedb_types::value_to_msgpack(&Value::Object(fields)).ok() - }) - .collect(); + } + fields + .entry("_surrogate".to_string()) + .or_insert(raw_identity); + let row = nodedb_types::value_to_msgpack(&Value::Object(fields)).map_err(|e| { + crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("vector hit row does not encode: {e}"), + } + })?; + rows.push(row); + } - encode_binary_rows(&rows) + Ok(encode_binary_rows(&rows)) +} + +/// One gathered vector-family hit in the Data-Plane wire shape. +#[derive(zerompk::FromMessagePack)] +#[msgpack(map)] +struct GatheredVectorHit { + id: crate::data::executor::handlers::hybrid_key::HybridFusionKey, + distance: f32, + doc_id: Option, + body: Option>, +} + +/// Whether a hit takes its key from the resolver: it has no body, or a whole +/// body with no `id` key. The scan reads the map header and each key, and +/// skips each value without decoding or allocating. A truncated body takes +/// no key: the flatten reports it as corruption. +fn body_needs_resolved_key(body: Option<&[u8]>) -> bool { + use nodedb_query::msgpack_scan::{extract_field, skip_value}; + + match body { + None => true, + Some(body) => { + skip_value(body, 0).is_some() + && extract_field(body, 0, nodedb_types::DEFAULT_IDENTITY_COLUMN).is_none() + } + } +} + +/// The surrogates of the bound hits in a gathered vector hit array, in hit +/// order, that [`flatten_vector_hits_to_relational_rows`] asks its resolver +/// for: those with no body, or a whole body with no `id`. Bodies are scanned, +/// never decoded. A payload that is not a hit array yields none, as the +/// flatten leaves it unchanged. +pub fn vector_hit_surrogates(bytes: &[u8]) -> Vec { + let hits: Vec = zerompk::from_msgpack(bytes).unwrap_or_default(); + hits.iter() + .filter(|h| body_needs_resolved_key(h.body.as_deref())) + .filter_map(|h| h.id.storage_key().map(|k| k.surrogate().as_u32())) + .collect() +} + +/// The surrogates of the bound hits in a gathered hybrid hit array, in hit +/// order: each `doc_id` storage key [`flatten_hybrid_hits_to_relational_rows`] +/// hands its resolver. A headless sentinel names no surrogate. A payload that +/// is not a hit array yields none. +pub fn hybrid_hit_surrogates(bytes: &[u8]) -> Vec { + use nodedb_types::Value; + + let Ok(Value::Array(hits)) = nodedb_types::value_from_msgpack(bytes) else { + return Vec::new(); + }; + hits.iter() + .filter_map(|hit| match hit { + Value::Object(fields) => match fields.get("doc_id") { + Some(Value::String(doc_id)) => { + nodedb_types::StorageKey::parse(doc_id).map(|key| key.surrogate().as_u32()) + } + _ => None, + }, + _ => None, + }) + .collect() } /// Flatten a gathered array of hybrid (RRF fusion) search hits into bare @@ -416,7 +492,7 @@ mod tests { .unwrap(); // Body carries `id`, so the resolver is not consulted. - let rows = decode_rows(&flatten_vector_hits_to_relational_rows(&input, |_| None)); + let rows = decode_rows(&flatten_vector_hits_to_relational_rows(&input, |_| None).unwrap()); assert_eq!(rows.len(), 1); let row = &rows[0]; assert_eq!(row.get("id"), Some(&Value::String("r0".into()))); @@ -437,9 +513,9 @@ mod tests { }]) .unwrap(); - let rows = decode_rows(&flatten_vector_hits_to_relational_rows(&input, |s| { - Some(format!("pk-{s}")) - })); + let rows = decode_rows( + &flatten_vector_hits_to_relational_rows(&input, |s| Some(format!("pk-{s}"))).unwrap(), + ); assert_eq!(rows.len(), 1); assert_eq!(rows[0].get("id"), Some(&Value::String("pk-42".into()))); assert_eq!(rows[0].get("_surrogate"), Some(&Value::Integer(42))); @@ -456,7 +532,7 @@ mod tests { }]) .unwrap(); - let rows = decode_rows(&flatten_vector_hits_to_relational_rows(&input, |_| None)); + let rows = decode_rows(&flatten_vector_hits_to_relational_rows(&input, |_| None).unwrap()); assert_eq!(rows[0].get("id"), Some(&Value::Integer(9))); } @@ -485,4 +561,106 @@ mod tests { Some(&Value::String("0000000a".into())) ); } + + fn hit(surrogate: u32, body: Option>) -> TestHit { + TestHit { + id: test_storage_key(surrogate), + distance: 0.1, + doc_id: None, + body, + } + } + + /// Only the hits the flatten names from the resolver are listed: a body + /// with an `id` is left out, a body without one and no body are listed. + #[test] + fn vector_hit_surrogates_lists_the_hits_the_flatten_resolves() { + let input = zerompk::to_msgpack_vec(&vec![ + hit(9, Some(bare_doc(vec![("id", Value::String("r9".into()))]))), + hit(5, Some(bare_doc(vec![("tag", Value::String("t".into()))]))), + TestHit { + id: "__local_3".into(), + distance: 0.2, + doc_id: None, + body: None, + }, + hit(4, None), + ]) + .unwrap(); + assert_eq!(vector_hit_surrogates(&input), vec![5, 4]); + assert!(vector_hit_surrogates(b"not msgpack hits").is_empty()); + + // The flatten asks the resolver for exactly the listed surrogates. + let asked = std::cell::RefCell::new(Vec::new()); + flatten_vector_hits_to_relational_rows(&input, |s| { + asked.borrow_mut().push(s); + None + }) + .unwrap(); + assert_eq!(asked.into_inner(), vec![5, 4]); + } + + /// A truncated stored body is corruption: the flatten refuses it, and + /// `vector_hit_surrogates` asks no key for it. + #[test] + fn a_truncated_body_fails_the_flatten() { + let mut body = bare_doc(vec![("tag", Value::String("keep".into()))]); + body.truncate(body.len() - 2); + let input = zerompk::to_msgpack_vec(&vec![hit(6, Some(body))]).unwrap(); + assert!(vector_hit_surrogates(&input).is_empty()); + let err = flatten_vector_hits_to_relational_rows(&input, |_| Some("resolved".into())) + .expect_err("a truncated body fails the flatten"); + assert!( + matches!(err, crate::Error::Serialization { .. }), + "corruption is a serialization error: {err:?}" + ); + } + + /// An `id` key after a nested map value is still found. + #[test] + fn an_id_after_a_nested_map_is_detected() { + let nested = Value::Object( + [("inner".to_string(), Value::Integer(1))] + .into_iter() + .collect(), + ); + // A fixmap of two pairs, written by hand so `meta` precedes `id`. + let mut body = vec![0x82u8]; + for (key, value) in [("meta", nested), ("id", Value::String("r7".into()))] { + body.extend(nodedb_types::value_to_msgpack(&Value::String(key.into())).unwrap()); + body.extend(nodedb_types::value_to_msgpack(&value).unwrap()); + } + assert!(!body_needs_resolved_key(Some(body.as_slice()))); + let without_id = bare_doc(vec![("x", Value::Integer(1))]); + assert!(body_needs_resolved_key(Some(without_id.as_slice()))); + assert!(body_needs_resolved_key(None)); + + let input = zerompk::to_msgpack_vec(&vec![hit(7, Some(body))]).unwrap(); + assert!(vector_hit_surrogates(&input).is_empty()); + let rows = decode_rows( + &flatten_vector_hits_to_relational_rows(&input, |_| Some("resolved".into())).unwrap(), + ); + assert_eq!(rows[0].get("id"), Some(&Value::String("r7".into()))); + } + + #[test] + fn hybrid_hit_surrogates_skips_the_headless_sentinel() { + let hit = |doc_id: &str| { + Value::Object( + [ + ("doc_id".to_string(), Value::String(doc_id.into())), + ("score".to_string(), Value::Float(0.5)), + ] + .into_iter() + .collect(), + ) + }; + let input = nodedb_types::value_to_msgpack(&Value::Array(vec![ + hit("0000000a"), + hit("__local_2"), + hit("0000000b"), + ])) + .unwrap(); + assert_eq!(hybrid_hit_surrogates(&input), vec![10, 11]); + } } diff --git a/nodedb/src/data/executor/scan_normalize.rs b/nodedb/src/data/executor/scan_normalize.rs index a20bcf173..54969627a 100644 --- a/nodedb/src/data/executor/scan_normalize.rs +++ b/nodedb/src/data/executor/scan_normalize.rs @@ -199,21 +199,20 @@ impl CoreLoop { } // 3. Sparse/document engine (schemaless + strict + vector-primary - // sidecar) — TRULY streams row-at-a-time via - // `scan_documents_for_each`. The body encoding is resolved once up + // sidecar) — streams via `for_each_current_row`: row-at-a-time on + // the plain table, a page at a time on a bitemporal collection's + // current versions. The body encoding is resolved once up // front through the same helper `scan_sparse` uses, so the streaming // and materializing scans cannot disagree about a row's shape. let database_id = crate::types::DatabaseId::new(did); let tenant_id = crate::types::TenantId::new(tid); let format = self.sparse_body_format(database_id, tenant_id, collection); let identity_column = self.identity_column(did, tid, collection); - self.sparse - .scan_documents_for_each(did, tid, collection, usize::MAX, |id, raw| { - let (id_s, mp) = - sparse_row_to_doc(id, raw, format.as_format_ref(), &identity_column); - f(&id_s, &mp) - })?; - Ok(()) + // Each row at its current version, so a bitemporal collection scans. + self.for_each_current_row(did, tid, collection, |id, raw| { + let (id_s, mp) = sparse_row_to_doc(id, raw, format.as_format_ref(), &identity_column); + f(&id_s, &mp).map(|()| std::ops::ControlFlow::Continue(())) + }) } /// Scan KV engine entries → standard msgpack. @@ -316,7 +315,8 @@ impl CoreLoop { collection: &str, limit: usize, ) -> crate::Result)>> { - let docs = self.sparse.scan_documents(did, tid, collection, limit)?; + // Each row at its current version, so a bitemporal collection scans. + let docs = self.current_rows(did, tid, collection, limit)?; let database_id = crate::types::DatabaseId::new(did); let tenant_id = crate::types::TenantId::new(tid); let format = self.sparse_body_format(database_id, tenant_id, collection); diff --git a/nodedb/src/diag/context/data_plane.rs b/nodedb/src/diag/context/data_plane.rs index 2d64dc04d..546f10793 100644 --- a/nodedb/src/diag/context/data_plane.rs +++ b/nodedb/src/diag/context/data_plane.rs @@ -184,3 +184,73 @@ impl DomainContext for CoreFailStopped<'_> { }) } } + +/// A Calvin participant refused its slice of a staged transaction with an +/// internal error. The verdict hands the error to the coordinator's client +/// only. +pub(in crate::diag) struct CalvinStageRefusedInternal<'a> { + pub vshard_id: u32, + pub epoch: u64, + pub position: u32, + pub detail: &'a str, +} + +impl DomainContext for CalvinStageRefusedInternal<'_> { + fn domain_kind(&self) -> &'static str { + "nodedb.calvin_stage_refused_internal" + } + + fn grouping_key(&self) -> String { + // The detail names the broken invariant before its first quoted name + // or colon. The collection, vShard and txn are the occurrence. + let class = self.detail.split(['\'', ':']).next().unwrap_or(self.detail); + format!("cause={}", class.trim()) + } + + fn to_json(&self) -> Value { + json!({ + "vshard_id": self.vshard_id, + "epoch": self.epoch, + "position": self.position, + "detail": self.detail, + "why_fatal": "an internal stage error is an invariant break no client input \ + causes. The transaction aborts on every participant, and only the \ + coordinator's client sees the text, so the refusing node keeps no \ + other record of it", + "operator_action": "the detail names the plan shape the stage refused. It is a \ + coordinator-side planning defect, not a storage fault", + }) + } +} + +/// A vector delete found the row behind the vector, but its stored payload +/// did not decode, so the payload index entries of the row cannot be found. +pub(in crate::diag) struct VectorPayloadUndecodable<'a> { + pub collection: &'a str, + pub detail: &'a str, +} + +impl DomainContext for VectorPayloadUndecodable<'_> { + fn domain_kind(&self) -> &'static str { + "nodedb.vector_payload_undecodable" + } + + fn grouping_key(&self) -> String { + // One root cause: a stored payload the decoder refuses. The + // collection and the decoder text are the occurrence. + "payload=undecodable".to_string() + } + + fn to_json(&self) -> Value { + json!({ + "collection": self.collection, + "detail": self.detail, + "why_fatal": "the payload index keys each vector by the row's field values. \ + Without the decoded row, its entries cannot be removed, so the \ + delete is refused and leaves the vector and its row in place", + "operator_action": "the stored row of the named collection is corrupt or was \ + written in another encoding. Read the row and rewrite it, \ + then repeat the delete", + }) + } +} diff --git a/nodedb/src/diag/context/mod.rs b/nodedb/src/diag/context/mod.rs index 4b24aeb5b..037aa2e73 100644 --- a/nodedb/src/diag/context/mod.rs +++ b/nodedb/src/diag/context/mod.rs @@ -52,7 +52,8 @@ pub(in crate::diag) use crdt_dead_letter_store::{ pub(in crate::diag) use cut_barrier::CutBarrierNotPlaced; pub use data_plane::LostResponseWrite; pub(in crate::diag) use data_plane::{ - CalvinApplyHalted, CalvinCompletionTimeout, CoreFailStopped, DataPlaneResponseLost, + CalvinApplyHalted, CalvinCompletionTimeout, CalvinStageRefusedInternal, CoreFailStopped, + DataPlaneResponseLost, VectorPayloadUndecodable, }; pub(in crate::diag) use event_image::{StrictRowImageUnrendered, TimeseriesRowImageUndecodable}; pub(in crate::diag) use held_entry::{ApplyPanicked, HeldEntryDebtNotRecorded, RedoEntryMalformed}; diff --git a/nodedb/src/diag/mod.rs b/nodedb/src/diag/mod.rs index f7b307952..42199c437 100644 --- a/nodedb/src/diag/mod.rs +++ b/nodedb/src/diag/mod.rs @@ -13,14 +13,14 @@ pub use recording::{ IndexRebuildTarget, VectorBuildTarget, apply_panicked, array_snapshot_buffer_poisoned, audit_flush_not_persisted, audit_wal_append_failed, batch_insert_without_surrogates, calvin_apply_halted, calvin_barrier_log_store_failed, calvin_completion_timeout, - calvin_left_state_not_forgotten, catalog_apply_orphan_row, collection_purge_row_missing, - columnar_segment_corrupt, constraint_install_deferred, consumer_group_offsets_retained, - continuous_aggregate_not_applied, continuous_aggregate_state_fault, - crdt_dead_letter_not_enqueued, crdt_dead_letter_not_restored, crdt_dead_letter_not_stored, - cut_barrier_not_placed, data_plane_core_fail_stopped, data_plane_response_lost, - data_plane_responses_lost, descriptor_lease_not_renewed, entry_kind, fts_index_update_failed, - held_entry_debt_not_recorded, history_compaction_not_applied, ilp_invalid_utf8_drop, - ilp_line_read_drop, index_rebuild_not_installed, metadata_apply_wedged, + calvin_left_state_not_forgotten, calvin_stage_refused_internal, catalog_apply_orphan_row, + collection_purge_row_missing, columnar_segment_corrupt, constraint_install_deferred, + consumer_group_offsets_retained, continuous_aggregate_not_applied, + continuous_aggregate_state_fault, crdt_dead_letter_not_enqueued, crdt_dead_letter_not_restored, + crdt_dead_letter_not_stored, cut_barrier_not_placed, data_plane_core_fail_stopped, + data_plane_response_lost, data_plane_responses_lost, descriptor_lease_not_renewed, entry_kind, + fts_index_update_failed, held_entry_debt_not_recorded, history_compaction_not_applied, + ilp_invalid_utf8_drop, ilp_line_read_drop, index_rebuild_not_installed, metadata_apply_wedged, orphaned_index_entry_after_delete, quota_row_invalid, quota_row_undecodable, quota_row_write_failed, quota_scope_purge_incomplete, quota_scope_replay_aborted, raft_entries_reapplied, raft_entry_reapplied, redo_abandon_given_up, redo_entry_malformed, @@ -29,7 +29,7 @@ pub use recording::{ strict_row_undecodable, synonym_group_not_applied, timeseries_flush_not_installed, timeseries_idle_flush_failed, timeseries_partition_unreadable, timeseries_row_image_undecodable, vector_build_failed, vector_builder_disconnected, - vector_builder_spawn_failed, vector_index_not_applied, vector_rebuild_unreadable, - wal_archival_failed_truncation_held, write_acked_without_durability, write_window_held, - write_window_leaked, + vector_builder_spawn_failed, vector_index_not_applied, vector_payload_undecodable, + vector_rebuild_unreadable, wal_archival_failed_truncation_held, write_acked_without_durability, + write_window_held, write_window_leaked, }; diff --git a/nodedb/src/diag/recording/data_plane.rs b/nodedb/src/diag/recording/data_plane.rs index aad0d0552..aa2451f19 100644 --- a/nodedb/src/diag/recording/data_plane.rs +++ b/nodedb/src/diag/recording/data_plane.rs @@ -105,3 +105,35 @@ pub fn data_plane_core_fail_stopped(core_id: usize, cause: &str, detail: &str) { .with_backtrace() .emit(); } + +/// Report a Calvin participant that refused its slice with an internal +/// error. Called from `calvin_stage_refusal`, the one site every refused +/// stage passes through. +pub fn calvin_stage_refused_internal(vshard_id: u32, epoch: u64, position: u32, detail: &str) { + let ctx = context::CalvinStageRefusedInternal { + vshard_id, + epoch, + position, + detail, + }; + let _ = Capture::new( + EventKind::InvariantViolation, + "Calvin participant refused its slice with an internal error", + ) + .domain(&ctx) + .with_backtrace() + .emit(); +} + +/// Report a vector delete whose row payload did not decode. Called from the +/// vector delete handler, which refuses the delete with nothing removed. +pub fn vector_payload_undecodable(collection: &str, detail: &str) { + let ctx = context::VectorPayloadUndecodable { collection, detail }; + let _ = Capture::new( + EventKind::Corruption, + "vector delete refused: the row's stored payload does not decode", + ) + .domain(&ctx) + .with_backtrace() + .emit(); +} diff --git a/nodedb/src/diag/recording/mod.rs b/nodedb/src/diag/recording/mod.rs index 26258f36a..382b7483a 100644 --- a/nodedb/src/diag/recording/mod.rs +++ b/nodedb/src/diag/recording/mod.rs @@ -48,8 +48,9 @@ pub use crdt::{ }; pub use cut_barrier::cut_barrier_not_placed; pub use data_plane::{ - calvin_apply_halted, calvin_completion_timeout, data_plane_core_fail_stopped, - data_plane_response_lost, data_plane_responses_lost, + calvin_apply_halted, calvin_completion_timeout, calvin_stage_refused_internal, + data_plane_core_fail_stopped, data_plane_response_lost, data_plane_responses_lost, + vector_payload_undecodable, }; pub use event_image::{strict_row_image_unrendered, timeseries_row_image_undecodable}; pub use held_entry::{apply_panicked, held_entry_debt_not_recorded, redo_entry_malformed}; diff --git a/nodedb/src/engine/sparse/btree_index.rs b/nodedb/src/engine/sparse/btree_index.rs index e4dce70a9..dc015da87 100644 --- a/nodedb/src/engine/sparse/btree_index.rs +++ b/nodedb/src/engine/sparse/btree_index.rs @@ -124,39 +124,54 @@ impl SparseEngine { collection: &str, field: &str, ) -> crate::Result { - let prefix = format!( - "{}{field}:", - coll_prefix(database_id, tenant_id, collection) - ); - let end = format!("{prefix}\u{ffff}"); - let write_txn = self .db .begin_write() .map_err(|e| redb_err("write txn", e))?; - - let removed; - { - let mut table = write_txn - .open_table(INDEXES) - .map_err(|e| redb_err("open indexes", e))?; - - let keys_to_remove: Vec = table - .range(prefix.as_str()..end.as_str()) - .map_err(|e| redb_err("index range", e))? - .filter_map(|r| r.ok().map(|(k, _)| k.value().to_string())) - .collect(); - - removed = keys_to_remove.len(); - for key in &keys_to_remove { - table - .remove(key.as_str()) - .map_err(|e| redb_err("remove index entry", e))?; - } - } + let removed = self.delete_index_entries_for_field_in_txn( + &write_txn, + database_id, + tenant_id, + collection, + field, + )?; write_txn .commit() .map_err(|e| redb_err("commit index delete", e))?; + Ok(removed) + } + + /// [`Self::delete_index_entries_for_field`] inside a caller-owned write + /// transaction. Does NOT commit. A read error fails the removal. + pub fn delete_index_entries_for_field_in_txn( + &self, + txn: &WriteTransaction, + database_id: u64, + tenant_id: u64, + collection: &str, + field: &str, + ) -> crate::Result { + let prefix = format!( + "{}{field}:", + coll_prefix(database_id, tenant_id, collection) + ); + let end = format!("{prefix}\u{ffff}"); + + let mut table = txn + .open_table(INDEXES) + .map_err(|e| redb_err("open indexes", e))?; + let keys_to_remove: Vec = table + .range(prefix.as_str()..end.as_str()) + .map_err(|e| redb_err("index range", e))? + .map(|r| r.map(|(k, _)| k.value().to_string())) + .collect::>() + .map_err(|e| redb_err("index entry", e))?; + let removed = keys_to_remove.len(); + for key in &keys_to_remove { + table + .remove(key.as_str()) + .map_err(|e| redb_err("remove index entry", e))?; + } if removed > 0 { debug!( diff --git a/nodedb/src/engine/sparse/btree_versioned/doc.rs b/nodedb/src/engine/sparse/btree_versioned/doc.rs index fc39de290..535e70889 100644 --- a/nodedb/src/engine/sparse/btree_versioned/doc.rs +++ b/nodedb/src/engine/sparse/btree_versioned/doc.rs @@ -290,6 +290,86 @@ impl SparseEngine { ) -> crate::Result>> { self.versioned_get_as_of(database_id, tenant, coll, doc_id, None, None) } + + /// Visit every version of every row of `coll`, in row order and, within + /// a row, in ascending system time. A live version hands over its body. + /// A tombstone or an erasure hands over `None`. A visit error ends the + /// scan and is returned. + pub fn versioned_for_each_version( + &self, + database_id: u64, + tenant: u64, + coll: &str, + mut visit: impl FnMut(&StorageKey, i64, Option<&[u8]>) -> crate::Result<()>, + ) -> crate::Result<()> { + let lo = super::key::coll_prefix(database_id, tenant, coll); + let hi = super::key::coll_prefix_end(database_id, tenant, coll); + let txn = self.db.begin_read().map_err(|e| redb_err("read txn", e))?; + let t = txn + .open_table(DOCUMENTS_VERSIONED) + .map_err(|e| redb_err("open table", e))?; + let range = t + .range(lo.as_str()..hi.as_str()) + .map_err(|e| redb_err("range", e))?; + for entry in range { + let (k, v) = entry.map_err(|e| redb_err("entry", e))?; + let key_str = k.value(); + let Some((id_part, suffix)) = key_str.rsplit_once('\x00') else { + continue; + }; + let Some(seg) = id_part.strip_prefix(lo.as_str()) else { + continue; + }; + let doc_id = StorageKey::parse(seg).ok_or_else(|| { + crate::engine::sparse::btree::invalid_storage_key_err( + crate::engine::sparse::btree::KeyedTable::DocumentsVersioned, + coll, + seg, + ) + })?; + let Ok(sys_from_ms) = suffix.parse::() else { + continue; + }; + let bytes = v.value(); + let decoded = decode_value(bytes)?; + visit( + &doc_id, + sys_from_ms, + decoded.is_live().then_some(decoded.body), + )?; + } + Ok(()) + } + + /// [`Self::versioned_get_current`] read inside `txn`, so it sees the + /// versions `txn` staged. The newest version decides: a tombstone or an + /// erasure hides the row. + pub fn versioned_get_current_in_txn( + &self, + txn: &redb::WriteTransaction, + database_id: u64, + tenant: u64, + coll: &str, + doc_id: &StorageKey, + ) -> crate::Result>> { + let lo = doc_prefix(database_id, tenant, coll, doc_id); + let hi = doc_prefix_end(database_id, tenant, coll, doc_id); + let t = txn + .open_table(DOCUMENTS_VERSIONED) + .map_err(|e| redb_err("open table", e))?; + let newest = t + .range(lo.as_str()..hi.as_str()) + .map_err(|e| redb_err("range", e))? + .next_back() + .transpose() + .map_err(|e| redb_err("entry", e))?; + let Some((_key, value)) = newest else { + return Ok(None); + }; + let bytes = value.value(); + let decoded = decode_value(bytes)?; + Ok(decoded.is_live().then(|| decoded.body.to_vec())) + } } #[cfg(test)] diff --git a/nodedb/src/engine/sparse/btree_versioned/index.rs b/nodedb/src/engine/sparse/btree_versioned/index.rs index c286c8f10..3a8b32397 100644 --- a/nodedb/src/engine/sparse/btree_versioned/index.rs +++ b/nodedb/src/engine/sparse/btree_versioned/index.rs @@ -105,6 +105,39 @@ impl SparseEngine { Ok(()) } + /// Remove every versioned index entry of `field` in `coll`: each value, + /// each row and each system time, tombstones included. A dropped index + /// leaves nothing, so a later index on the same field starts clean. + /// + /// Runs inside a caller-owned write transaction and does NOT commit. + /// Returns the number of entries removed. + pub fn versioned_index_drop_field_in_txn( + &self, + txn: &redb::WriteTransaction, + database_id: u64, + tenant: u64, + coll: &str, + field: &str, + ) -> crate::Result { + let lo = format!("{database_id}:{tenant}:{coll}:{field}:"); + // `:` = 0x3A, next byte `;` = 0x3B gives a clean exclusive bound. + let hi = format!("{database_id}:{tenant}:{coll}:{field};"); + let mut t = txn + .open_table(INDEXES_VERSIONED) + .map_err(|e| redb_err("open table", e))?; + let keys: Vec = t + .range(lo.as_str()..hi.as_str()) + .map_err(|e| redb_err("range", e))? + .map(|r| r.map(|(k, _)| k.value().to_string())) + .collect::>() + .map_err(|e| redb_err("entry", e))?; + for key in &keys { + t.remove(key.as_str()) + .map_err(|e| redb_err("remove entry", e))?; + } + Ok(keys.len()) + } + /// Look up doc_ids for a `(field, value)` pair at a system-time cutoff. /// Returns only doc_ids whose newest entry ≤ cutoff is live. pub fn versioned_index_lookup_as_of( diff --git a/nodedb/src/event/consumer/run.rs b/nodedb/src/event/consumer/run.rs index 02da13025..702fa8837 100644 --- a/nodedb/src/event/consumer/run.rs +++ b/nodedb/src/event/consumer/run.rs @@ -575,7 +575,15 @@ mod tests { /// Run `WRITES` logged writes through a consumer whose ring holds /// `ring_capacity` events, and return what the side effects saw, the /// number of ring drops, and the persisted watermark after shutdown. + /// + /// The caller runs on a current-thread runtime. The consumer task then + /// runs only while this function awaits, never during the write burst. async fn run(ring_capacity: usize) -> (Totals, u64, Lsn, Lsn) { + assert_eq!( + tokio::runtime::Handle::current().runtime_flavor(), + tokio::runtime::RuntimeFlavor::CurrentThread, + "the burst must run with the consumer parked" + ); let dir = tempfile::tempdir().expect("tempdir"); let (wal, watermark_store, shared, trigger_dlq, cdc_router) = crate::event::test_utils::event_test_deps(&dir); @@ -600,13 +608,15 @@ mod tests { slab_account: Arc::new(crate::event::slab_budget::ConsumerSlabAccount::new(0)), }); - // Every write is logged, durable and final before its event leaves, - // as on the write path. The burst outruns a small ring, which drops events the - // consumer then rebuilds from the WAL while the ring still carries - // the rest: some events reach the consumer on both paths. - // Let the idle consumer ramp to its long poll sleep, so the burst - // below lands while it sleeps. + // The consumer starts and reads its boot WAL bound while this sleeps. + // So the burst below lands mid-run, above that bound. tokio::time::sleep(Duration::from_millis(200)).await; + // Every write is logged, durable and final before its event leaves, + // as on the write path. The burst has no await point, so the parked + // consumer takes nothing from the ring until it ends. A small ring + // keeps its first events and drops the rest. The consumer then takes + // the kept events and rebuilds every write from the WAL: the kept + // events reach it on both paths. let mut dropped = 0u64; let mut last_lsn = Lsn::ZERO; for i in 1..=WRITES { @@ -661,7 +671,7 @@ mod tests { /// An event carried by both the ring and WAL catch-up reaches every side /// effect once, and a dropped event reaches each of them all the same. - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[tokio::test(flavor = "current_thread")] async fn catch_up_mid_run_delivers_every_event_exactly_once() { let expected = Totals { audit_rows: WRITES as usize, diff --git a/nodedb/tests/inproc/cases/array_sql_relation.rs b/nodedb/tests/inproc/cases/array_sql_relation.rs index daa30964c..9cb78edda 100644 --- a/nodedb/tests/inproc/cases/array_sql_relation.rs +++ b/nodedb/tests/inproc/cases/array_sql_relation.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 -//! `ARRAY_SLICE` as a relation: every SELECT clause applies over its rows. +//! `ARRAY_SLICE` and `ARRAY_AGG` as relations: every SELECT clause applies +//! over their rows. //! //! The fixture is a 4x4 image with two bands. Cell `(x, y)` has index //! `k = 4x + y`, `nir = (k + 1) / 20` and `red = (16 - k) / 40`, so every @@ -253,18 +254,26 @@ async fn array_slice_derived_table_matches_direct() { } #[tokio::test] -async fn array_agg_where_refused() { +async fn array_agg_result_rows_take_where_and_order_by() { let srv = TestServer::start().await; prepare_img(&srv).await; - srv.expect_error( - "SELECT * FROM ARRAY_AGG('img', 'nir', 'sum') WHERE result > 1", - "WHERE over ARRAY_AGG is not supported", - ) - .await; - srv.expect_error( - "SELECT * FROM ARRAY_AGG('img', 'nir', 'sum') ORDER BY result", - "ORDER BY over ARRAY_AGG is not supported", - ) - .await; + // Group `x` sums to `(16x + 10) / 20`: 0.5, 1.3, 2.1, 2.9. + let sql = "SELECT \"group\", ROUND(result, 1) AS total \ + FROM ARRAY_AGG('img', 'nir', 'sum', 'x') \ + WHERE result > 1 ORDER BY \"group\" DESC"; + let rows = srv + .query_rows(sql) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")); + let got: Vec<(i64, f64)> = rows + .iter() + .map(|row| (int(&row[0]), float(&row[1]))) + .collect(); + let expected = [(3, 2.9), (2, 2.1), (1, 1.3)]; + assert_eq!(got.len(), expected.len(), "{got:?}"); + for ((g, v), (eg, ev)) in got.iter().zip(expected) { + assert_eq!(*g, eg, "{got:?}"); + assert!((v - ev).abs() < 1e-9, "{got:?}"); + } } diff --git a/nodedb/tests/wire/cases/array_agg_relation.rs b/nodedb/tests/wire/cases/array_agg_relation.rs new file mode 100644 index 000000000..70a6d2d81 --- /dev/null +++ b/nodedb/tests/wire/cases/array_agg_relation.rs @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `ARRAY_AGG` and `ARRAY_ELEMENTWISE` as relations: every SELECT clause +//! applies over the rows the call emits, the same as over a derived table +//! that reads the call. +//! +//! The fixture is a 4x4 array `agr` with dims `t`, `x`. Cell `(t, x)` holds +//! `nir = (4t + x + 1) / 20`, so the mean of group `t` is `(4t + 2.5) / 20` +//! and the ungrouped sum is `6.8`. + +use crate::harness::TestServer; + +const GROUPED: &str = "ARRAY_AGG('agr', 'nir', 'mean', 't')"; + +async fn create_agr(server: &TestServer) { + server + .exec( + "CREATE ARRAY agr DIMS (t INT64 [0..3], x INT64 [0..3]) \ + ATTRS (nir FLOAT64) TILE_EXTENTS (2, 2)", + ) + .await + .expect("CREATE ARRAY agr"); + let cells: Vec = (0..4) + .flat_map(|t| { + (0..4).map(move |x| { + format!( + "COORDS ({t}, {x}) VALUES ({:?})", + (4 * t + x + 1) as f64 / 20.0 + ) + }) + }) + .collect(); + server + .exec(&format!("INSERT INTO ARRAY agr {}", cells.join(", "))) + .await + .expect("INSERT INTO ARRAY agr"); + server + .exec("SELECT ARRAY_FLUSH('agr')") + .await + .expect("ARRAY_FLUSH"); +} + +fn group_mean(t: i64) -> f64 { + (4 * t) as f64 / 20.0 + 2.5 / 20.0 +} + +fn number(text: &str) -> f64 { + text.parse() + .unwrap_or_else(|e| panic!("{text:?} is not a number: {e}")) +} + +/// `(group, value)` of every row `sql` returns, in row order. +async fn pairs(server: &TestServer, sql: &str) -> Vec<(i64, f64)> { + server + .query_rows(sql) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")) + .iter() + .map(|row| (number(&row[0]) as i64, number(&row[1]))) + .collect() +} + +fn assert_pairs(got: &[(i64, f64)], expected: &[(i64, f64)], sql: &str) { + assert_eq!(got.len(), expected.len(), "{sql}: {got:?}"); + for ((g, v), (eg, ev)) in got.iter().zip(expected) { + assert_eq!(g, eg, "{sql}: {got:?}"); + assert!((v - ev).abs() < 1e-9, "{sql}: {got:?}"); + } +} + +#[tokio::test] +async fn grouped_agg_orders_by_alias_with_a_rounded_result() { + let server = TestServer::start().await; + create_agr(&server).await; + let sql = format!( + "SELECT \"group\" AS date, ROUND(result, 4) AS mean_nir FROM {GROUPED} ORDER BY date DESC" + ); + let expected: Vec<(i64, f64)> = (0..4).rev().map(|t| (t, group_mean(t))).collect(); + assert_pairs(&pairs(&server, &sql).await, &expected, &sql); +} + +#[tokio::test] +async fn grouped_agg_orders_by_group_with_limit() { + let server = TestServer::start().await; + create_agr(&server).await; + let sql = format!("SELECT \"group\", result FROM {GROUPED} ORDER BY \"group\" LIMIT 2"); + assert_pairs( + &pairs(&server, &sql).await, + &[(0, group_mean(0)), (1, group_mean(1))], + &sql, + ); +} + +#[tokio::test] +async fn ungrouped_agg_rounds_its_result() { + let server = TestServer::start().await; + create_agr(&server).await; + let sql = "SELECT ROUND(result, 2) AS total FROM ARRAY_AGG('agr', 'nir', 'sum')"; + let rows = server + .query_named_rows(sql) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")); + assert_eq!(rows.len(), 1, "{sql}: one row: {rows:?}"); + assert!((number(&rows[0]["total"]) - 6.8).abs() < 1e-9, "{rows:?}"); +} + +#[tokio::test] +async fn where_on_result_keeps_matching_groups() { + let server = TestServer::start().await; + create_agr(&server).await; + let sql = + format!("SELECT \"group\", result FROM {GROUPED} WHERE result > 0.3 ORDER BY \"group\""); + let expected: Vec<(i64, f64)> = (1..4).map(|t| (t, group_mean(t))).collect(); + assert_pairs(&pairs(&server, &sql).await, &expected, &sql); +} + +#[tokio::test] +async fn count_over_grouped_agg_counts_the_kept_groups() { + let server = TestServer::start().await; + create_agr(&server).await; + let sql = format!("SELECT COUNT(*) AS n FROM {GROUPED} WHERE result > 0.3"); + let rows = server + .query_named_rows(&sql) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")); + assert_eq!(rows.len(), 1, "{sql}: {rows:?}"); + assert_eq!(number(&rows[0]["n"]), 3.0, "{rows:?}"); +} + +#[tokio::test] +async fn direct_read_matches_the_derived_table() { + let server = TestServer::start().await; + create_agr(&server).await; + let direct = format!( + "SELECT \"group\" AS date, ROUND(result, 4) FROM {GROUPED} \ + WHERE result < 0.6 ORDER BY date DESC LIMIT 2" + ); + let derived = format!( + "SELECT a.\"group\" AS date, ROUND(a.result, 4) FROM (SELECT * FROM {GROUPED}) a \ + WHERE a.result < 0.6 ORDER BY date DESC LIMIT 2" + ); + let direct_rows = pairs(&server, &direct).await; + assert_eq!(direct_rows.len(), 2, "{direct}: {direct_rows:?}"); + assert_eq!(direct_rows, pairs(&server, &derived).await); +} + +#[tokio::test] +async fn count_over_elementwise_counts_its_rows() { + let server = TestServer::start().await; + create_agr(&server).await; + let sql = "SELECT COUNT(*) AS n FROM ARRAY_ELEMENTWISE('agr', 'agr', 'add', 'nir')"; + let rows = server + .query_named_rows(sql) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")); + assert_eq!(rows.len(), 1, "{sql}: {rows:?}"); + assert_eq!(number(&rows[0]["n"]), 16.0, "{rows:?}"); +} diff --git a/nodedb/tests/wire/cases/bitemporal_index_convert.rs b/nodedb/tests/wire/cases/bitemporal_index_convert.rs new file mode 100644 index 000000000..83b2b0651 --- /dev/null +++ b/nodedb/tests/wire/cases/bitemporal_index_convert.rs @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Secondary indexes and CONVERT on a bitemporal collection. +//! +//! - `DROP INDEX` removes the versioned index entries, so a later +//! `CREATE INDEX` on the same field starts clean. +//! - An `AS OF SYSTEM TIME` read with an indexed equality reads the versioned +//! index at its cutoff, so it returns the rows as they stood then. That +//! holds for an index created after the rows changed, too. +//! - `CONVERT` rewrites every version, so the current read and the history +//! read return the same rows after it. A target that holds no row versions +//! is refused with nothing moved. + +use crate::harness::TestServer; + +/// The wall-clock millisecond strictly between the writes before and after +/// it. +async fn instant_between_writes() -> i64 { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system time before epoch") + .as_millis(); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + i64::try_from(now).expect("millisecond clock fits i64") +} + +/// `(id, kind)` of every row `sql` returns, sorted by id. +async fn id_kind_rows(srv: &TestServer, sql: &str) -> Vec<(String, String)> { + let mut rows: Vec<(String, String)> = srv + .query_rows(sql) + .await + .unwrap_or_else(|e| panic!("run {sql}: {e}")) + .into_iter() + .map(|row| (row[0].clone(), row[1].clone())) + .collect(); + rows.sort(); + rows +} + +fn rows(expected: &[(&str, &str)]) -> Vec<(String, String)> { + expected + .iter() + .map(|(id, kind)| (id.to_string(), kind.to_string())) + .collect() +} + +async fn insert(srv: &TestServer, name: &str, id: &str, kind: &str) { + srv.exec(&format!( + "INSERT INTO {name} {{ id: '{id}', kind: '{kind}' }}" + )) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_recreated_index_holds_only_the_live_rows() { + let srv = TestServer::start().await; + srv.exec("CREATE COLLECTION bt_idx_drop WITH (bitemporal=true)") + .await + .unwrap(); + srv.exec("CREATE INDEX bt_idx_drop_kind ON bt_idx_drop(kind)") + .await + .unwrap(); + insert(&srv, "bt_idx_drop", "a", "x").await; + insert(&srv, "bt_idx_drop", "b", "y").await; + srv.exec("DROP INDEX bt_idx_drop_kind").await.unwrap(); + + // Writes with no index: a stale entry from the first index names the + // old values of a and b. + srv.exec("UPDATE bt_idx_drop SET kind = 'z' WHERE id = 'a'") + .await + .unwrap(); + srv.exec("DELETE FROM bt_idx_drop WHERE id = 'b'") + .await + .unwrap(); + insert(&srv, "bt_idx_drop", "c", "x").await; + srv.exec("CREATE INDEX bt_idx_drop_kind ON bt_idx_drop(kind)") + .await + .unwrap(); + + let lookup = + |kind: &'static str| format!("SELECT id, kind FROM bt_idx_drop WHERE kind = '{kind}'"); + assert_eq!(id_kind_rows(&srv, &lookup("x")).await, rows(&[("c", "x")])); + assert_eq!(id_kind_rows(&srv, &lookup("z")).await, rows(&[("a", "z")])); + assert!(id_kind_rows(&srv, &lookup("y")).await.is_empty()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn as_of_with_an_indexed_equality_reads_the_versions() { + let srv = TestServer::start().await; + srv.exec("CREATE COLLECTION bt_idx_asof WITH (bitemporal=true)") + .await + .unwrap(); + srv.exec("CREATE INDEX bt_idx_asof_kind ON bt_idx_asof(kind)") + .await + .unwrap(); + insert(&srv, "bt_idx_asof", "a", "x").await; + let before = instant_between_writes().await; + srv.exec("UPDATE bt_idx_asof SET kind = 'y' WHERE id = 'a'") + .await + .unwrap(); + + let as_of = |kind: &str| { + format!("SELECT id, kind FROM bt_idx_asof AS OF SYSTEM TIME {before} WHERE kind = '{kind}'") + }; + assert_eq!( + id_kind_rows(&srv, &as_of("x")).await, + rows(&[("a", "x")]), + "AS OF before the UPDATE finds the old value" + ); + assert!(id_kind_rows(&srv, &as_of("y")).await.is_empty()); + assert_eq!( + id_kind_rows(&srv, "SELECT id, kind FROM bt_idx_asof WHERE kind = 'y'").await, + rows(&[("a", "y")]) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn as_of_before_the_index_was_created_reads_the_versions() { + let srv = TestServer::start().await; + srv.exec("CREATE COLLECTION bt_idx_late WITH (bitemporal=true)") + .await + .unwrap(); + insert(&srv, "bt_idx_late", "a", "x").await; + insert(&srv, "bt_idx_late", "b", "y").await; + insert(&srv, "bt_idx_late", "c", "x").await; + let t1 = instant_between_writes().await; + srv.exec("UPDATE bt_idx_late SET kind = 'z' WHERE id = 'a'") + .await + .unwrap(); + srv.exec("DELETE FROM bt_idx_late WHERE id = 'c'") + .await + .unwrap(); + let t2 = instant_between_writes().await; + insert(&srv, "bt_idx_late", "d", "x").await; + // The backfill writes each version's entries at that version's time. + srv.exec("CREATE INDEX bt_idx_late_kind ON bt_idx_late(kind)") + .await + .unwrap(); + + let as_of = |at: i64, kind: &str| { + format!("SELECT id, kind FROM bt_idx_late AS OF SYSTEM TIME {at} WHERE kind = '{kind}'") + }; + assert_eq!( + id_kind_rows(&srv, &as_of(t1, "x")).await, + rows(&[("a", "x"), ("c", "x")]), + "before the update and the delete, a and c hold x" + ); + assert_eq!( + id_kind_rows(&srv, &as_of(t1, "y")).await, + rows(&[("b", "y")]) + ); + assert!( + id_kind_rows(&srv, &as_of(t2, "x")).await.is_empty(), + "after the update and the delete, before d, no row holds x" + ); + assert_eq!( + id_kind_rows(&srv, &as_of(t2, "z")).await, + rows(&[("a", "z")]) + ); + assert_eq!( + id_kind_rows(&srv, "SELECT id, kind FROM bt_idx_late WHERE kind = 'x'").await, + rows(&[("d", "x")]) + ); +} + +/// A semi-join whose subquery side is an indexed equality on a bitemporal +/// collection. Its bitmap prefilter reads the versioned index at the +/// statement's cutoff, so the side holds the rows as they stood then. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn as_of_join_side_with_an_indexed_equality_reads_the_versions() { + let srv = TestServer::start().await; + srv.exec("CREATE COLLECTION bt_cust WITH (bitemporal=true)") + .await + .unwrap(); + srv.exec("CREATE INDEX bt_cust_tier ON bt_cust(tier)") + .await + .unwrap(); + srv.exec("CREATE COLLECTION bt_orders WITH (bitemporal=true)") + .await + .unwrap(); + srv.exec("INSERT INTO bt_cust { id: 'c1', tier: 'gold' }") + .await + .unwrap(); + srv.exec("INSERT INTO bt_cust { id: 'c2', tier: 'silver' }") + .await + .unwrap(); + srv.exec("INSERT INTO bt_orders { id: 'o1', cust: 'c1' }") + .await + .unwrap(); + srv.exec("INSERT INTO bt_orders { id: 'o2', cust: 'c2' }") + .await + .unwrap(); + let before = instant_between_writes().await; + srv.exec("UPDATE bt_cust SET tier = 'silver' WHERE id = 'c1'") + .await + .unwrap(); + srv.exec("UPDATE bt_cust SET tier = 'gold' WHERE id = 'c2'") + .await + .unwrap(); + + let gold_orders = |as_of: &str| { + format!( + "SELECT id, cust FROM bt_orders {as_of} \ + WHERE cust IN (SELECT id FROM bt_cust WHERE tier = 'gold')" + ) + }; + assert_eq!( + id_kind_rows(&srv, &gold_orders(&format!("AS OF SYSTEM TIME {before}"))).await, + rows(&[("o1", "c1")]), + "before the updates, c1 was the gold customer" + ); + assert_eq!( + id_kind_rows(&srv, &gold_orders("")).await, + rows(&[("o2", "c2")]), + "now c2 is the gold customer" + ); +} + +/// The current rows and the rows `AS OF SYSTEM TIME` `before` of `name`. +async fn reads( + srv: &TestServer, + name: &str, + before: i64, +) -> (Vec<(String, String)>, Vec<(String, String)>) { + let current = id_kind_rows(srv, &format!("SELECT id, kind FROM {name}")).await; + let as_of = id_kind_rows( + srv, + &format!("SELECT id, kind FROM {name} AS OF SYSTEM TIME {before}"), + ) + .await; + (current, as_of) +} + +/// Seed `a` (`x`), `b` (`y`), `c` (`x`), then update `b` and delete `c`. +/// Returns an instant before the update and the delete. +async fn seed_history(srv: &TestServer, name: &str) -> i64 { + insert(srv, name, "a", "x").await; + insert(srv, name, "b", "y").await; + insert(srv, name, "c", "x").await; + let before = instant_between_writes().await; + srv.exec(&format!("UPDATE {name} SET kind = 'z' WHERE id = 'b'")) + .await + .unwrap(); + srv.exec(&format!("DELETE FROM {name} WHERE id = 'c'")) + .await + .unwrap(); + before +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn convert_to_strict_keeps_every_version() { + let srv = TestServer::start().await; + srv.exec("CREATE COLLECTION bt_conv_strict WITH (bitemporal=true)") + .await + .unwrap(); + let before = seed_history(&srv, "bt_conv_strict").await; + let expected = reads(&srv, "bt_conv_strict", before).await; + assert_eq!(expected.0, rows(&[("a", "x"), ("b", "z")])); + assert_eq!(expected.1, rows(&[("a", "x"), ("b", "y"), ("c", "x")])); + + srv.exec( + "CONVERT COLLECTION bt_conv_strict TO document_strict \ + (id TEXT PRIMARY KEY, kind TEXT)", + ) + .await + .unwrap(); + assert_eq!(reads(&srv, "bt_conv_strict", before).await, expected); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn convert_to_schemaless_keeps_every_version() { + let srv = TestServer::start().await; + srv.exec( + "CREATE COLLECTION bt_conv_doc (id STRING PRIMARY KEY, kind STRING) \ + WITH (engine='document_strict', bitemporal=true)", + ) + .await + .unwrap(); + let before = seed_history(&srv, "bt_conv_doc").await; + let expected = reads(&srv, "bt_conv_doc", before).await; + assert_eq!(expected.0, rows(&[("a", "x"), ("b", "z")])); + assert_eq!(expected.1, rows(&[("a", "x"), ("b", "y"), ("c", "x")])); + + srv.exec("CONVERT COLLECTION bt_conv_doc TO document_schemaless") + .await + .unwrap(); + assert_eq!(reads(&srv, "bt_conv_doc", before).await, expected); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn convert_to_kv_is_refused_with_nothing_moved() { + let srv = TestServer::start().await; + srv.exec("CREATE COLLECTION bt_conv_kv WITH (bitemporal=true)") + .await + .unwrap(); + let before = seed_history(&srv, "bt_conv_kv").await; + let expected = reads(&srv, "bt_conv_kv", before).await; + + srv.expect_error( + "CONVERT COLLECTION bt_conv_kv TO kv (id TEXT PRIMARY KEY, kind TEXT)", + "holds no row versions", + ) + .await; + assert_eq!(reads(&srv, "bt_conv_kv", before).await, expected); +} diff --git a/nodedb/tests/wire/cases/bitemporal_predicate_writes.rs b/nodedb/tests/wire/cases/bitemporal_predicate_writes.rs new file mode 100644 index 000000000..d936c231e --- /dev/null +++ b/nodedb/tests/wire/cases/bitemporal_predicate_writes.rs @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Predicate `DELETE` and `UPDATE`, `TRUNCATE` and facet counts on a +//! bitemporal collection. +//! +//! A bitemporal collection stores its rows only as versions. A predicate +//! write matches each row's current version. A `DELETE` or `TRUNCATE` closes +//! that version with a tombstone and an `UPDATE` writes a new one, so +//! `AS OF SYSTEM TIME` before the statement still returns the rows as they +//! were. Facet counts read the current versions. + +use tokio_postgres::SimpleQueryMessage; + +use crate::harness::TestServer; + +/// Create bitemporal `name` holding `a` (`x`), `b` (`y`) and `c` (`x`). +async fn seed(srv: &TestServer, name: &str) { + srv.exec(&format!("CREATE COLLECTION {name} WITH (bitemporal=true)")) + .await + .unwrap(); + for (id, kind) in [("a", "x"), ("b", "y"), ("c", "x")] { + srv.exec(&format!( + "INSERT INTO {name} {{ id: '{id}', kind: '{kind}' }}" + )) + .await + .unwrap(); + } +} + +/// The wall-clock millisecond strictly between the writes before and after +/// it. +async fn instant_between_writes() -> i64 { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system time before epoch") + .as_millis(); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + i64::try_from(now).expect("millisecond clock fits i64") +} + +/// The `CommandComplete` row count of the single statement `sql`. +async fn command_count(srv: &TestServer, sql: &str) -> u64 { + let counts: Vec = srv + .client + .simple_query(sql) + .await + .unwrap_or_else(|e| panic!("run {sql}: {e:?}")) + .into_iter() + .filter_map(|m| match m { + SimpleQueryMessage::CommandComplete(n) => Some(n), + _ => None, + }) + .collect(); + assert_eq!( + counts.len(), + 1, + "{sql} answers one command tag, got {counts:?}" + ); + counts[0] +} + +/// `(id, kind)` of every row `sql` returns, sorted by id. +async fn id_kind_rows(srv: &TestServer, sql: &str) -> Vec<(String, String)> { + let mut rows: Vec<(String, String)> = srv + .query_rows(sql) + .await + .unwrap_or_else(|e| panic!("run {sql}: {e}")) + .into_iter() + .map(|row| (row[0].clone(), row[1].clone())) + .collect(); + rows.sort(); + rows +} + +fn rows(expected: &[(&str, &str)]) -> Vec<(String, String)> { + expected + .iter() + .map(|(id, kind)| (id.to_string(), kind.to_string())) + .collect() +} + +/// The current rows of `name` and the rows `AS OF SYSTEM TIME` `before`. +async fn current_and_as_of( + srv: &TestServer, + name: &str, + before: i64, +) -> (Vec<(String, String)>, Vec<(String, String)>) { + let current = id_kind_rows(srv, &format!("SELECT id, kind FROM {name} ORDER BY id")).await; + let as_of = id_kind_rows( + srv, + &format!("SELECT id, kind FROM {name} AS OF SYSTEM TIME {before}"), + ) + .await; + (current, as_of) +} + +/// Autocommit: the predicate `DELETE` removes both matches and the +/// predicate `UPDATE` rewrites its match. History before each stays +/// readable. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn autocommit_predicate_writes_match_current_versions() { + let srv = TestServer::start().await; + seed(&srv, "bt_pw_auto").await; + + let before_delete = instant_between_writes().await; + assert_eq!( + command_count(&srv, "DELETE FROM bt_pw_auto WHERE kind = 'x'").await, + 2, + "DELETE matches the current versions of a and c" + ); + let (current, as_of) = current_and_as_of(&srv, "bt_pw_auto", before_delete).await; + assert_eq!(current, rows(&[("b", "y")]), "the deleted rows are gone"); + assert_eq!( + as_of, + rows(&[("a", "x"), ("b", "y"), ("c", "x")]), + "AS OF before the DELETE returns the deleted rows" + ); + + let before_update = instant_between_writes().await; + assert_eq!( + command_count(&srv, "UPDATE bt_pw_auto SET kind = 'z' WHERE kind = 'y'").await, + 1, + "UPDATE matches the current version of b" + ); + let (current, as_of) = current_and_as_of(&srv, "bt_pw_auto", before_update).await; + assert_eq!(current, rows(&[("b", "z")]), "the update is current"); + assert_eq!( + as_of, + rows(&[("b", "y")]), + "AS OF before the UPDATE returns the old value" + ); +} + +/// The same writes inside `BEGIN ... COMMIT`. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn transaction_predicate_writes_match_current_versions() { + let srv = TestServer::start().await; + seed(&srv, "bt_pw_txn").await; + + let before_delete = instant_between_writes().await; + srv.exec("BEGIN").await.unwrap(); + assert_eq!( + command_count(&srv, "DELETE FROM bt_pw_txn WHERE kind = 'x'").await, + 2, + "DELETE matches the current versions of a and c" + ); + srv.exec("COMMIT").await.unwrap(); + let (current, as_of) = current_and_as_of(&srv, "bt_pw_txn", before_delete).await; + assert_eq!(current, rows(&[("b", "y")]), "the deleted rows are gone"); + assert_eq!( + as_of, + rows(&[("a", "x"), ("b", "y"), ("c", "x")]), + "AS OF before the DELETE returns the deleted rows" + ); + + let before_update = instant_between_writes().await; + srv.exec("BEGIN").await.unwrap(); + assert_eq!( + command_count(&srv, "UPDATE bt_pw_txn SET kind = 'z' WHERE kind = 'y'").await, + 1, + "UPDATE matches the current version of b" + ); + srv.exec("COMMIT").await.unwrap(); + let (current, as_of) = current_and_as_of(&srv, "bt_pw_txn", before_update).await; + assert_eq!(current, rows(&[("b", "z")]), "the update is current"); + assert_eq!( + as_of, + rows(&[("b", "y")]), + "AS OF before the UPDATE returns the old value" + ); +} + +/// Autocommit `TRUNCATE` closes every row. `AS OF SYSTEM TIME` before it +/// still returns them. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn autocommit_truncate_closes_every_current_version() { + let srv = TestServer::start().await; + seed(&srv, "bt_tr_auto").await; + + let before = instant_between_writes().await; + srv.exec("TRUNCATE bt_tr_auto").await.unwrap(); + let (current, as_of) = current_and_as_of(&srv, "bt_tr_auto", before).await; + assert!( + current.is_empty(), + "TRUNCATE leaves no current row, got {current:?}" + ); + assert_eq!( + as_of, + rows(&[("a", "x"), ("b", "y"), ("c", "x")]), + "AS OF before the TRUNCATE returns every row" + ); +} + +/// The same `TRUNCATE` inside `BEGIN ... COMMIT`. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn transaction_truncate_closes_every_current_version() { + let srv = TestServer::start().await; + seed(&srv, "bt_tr_txn").await; + + let before = instant_between_writes().await; + srv.exec("BEGIN").await.unwrap(); + srv.exec("TRUNCATE bt_tr_txn").await.unwrap(); + srv.exec("COMMIT").await.unwrap(); + let (current, as_of) = current_and_as_of(&srv, "bt_tr_txn", before).await; + assert!( + current.is_empty(), + "TRUNCATE leaves no current row, got {current:?}" + ); + assert_eq!( + as_of, + rows(&[("a", "x"), ("b", "y"), ("c", "x")]), + "AS OF before the TRUNCATE returns every row" + ); +} + +/// Facet counts read each row's current version: the rows a `DELETE` +/// closed no longer count. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn facet_counts_read_current_versions() { + let srv = TestServer::start().await; + seed(&srv, "bt_facet").await; + srv.exec("INSERT INTO bt_facet { id: 'd', kind: 'y' }") + .await + .unwrap(); + assert_eq!( + command_count(&srv, "DELETE FROM bt_facet WHERE id = 'a'").await, + 1 + ); + + let text = srv + .query_text_joined("SELECT FACET_COUNTS(collection => 'bt_facet', fields => ['kind'])") + .await + .unwrap() + .join(""); + let facets: serde_json::Value = + serde_json::from_str(&text).unwrap_or_else(|e| panic!("facet JSON {text}: {e}")); + let mut counts: Vec<(String, u64)> = facets["kind"] + .as_array() + .unwrap_or_else(|| panic!("a kind facet, got {facets}")) + .iter() + .map(|entry| { + ( + entry["value"].as_str().unwrap_or_default().to_string(), + entry["count"].as_u64().unwrap_or_default(), + ) + }) + .collect(); + counts.sort(); + assert_eq!( + counts, + vec![("x".to_string(), 1), ("y".to_string(), 2)], + "the deleted row a no longer counts" + ); +} diff --git a/nodedb/tests/wire/cases/calvin_txn_edge_update.rs b/nodedb/tests/wire/cases/calvin_txn_edge_update.rs new file mode 100644 index 000000000..e681c68d0 --- /dev/null +++ b/nodedb/tests/wire/cases/calvin_txn_edge_update.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Predicate writes inside a transaction that need more than the row set: +//! +//! - An UPDATE of an edge document's `_to` moves its mirrored graph edge in +//! the same transaction. The edge tasks derive from the statement's own +//! read, so the COMMIT takes the Calvin path with them. +//! - A predicate DELETE on a `bitemporal=true` collection reads and checks +//! the rows' current versions, so its Calvin COMMIT removes them. + +use crate::harness::TestServer; + +/// The destination nodes of `node`'s outgoing `l` edges in `collection`. +async fn neighbors_of(server: &TestServer, collection: &str, node: &str) -> Vec { + let rows = server + .query_text(&format!( + "GRAPH NEIGHBORS IN '{collection}' OF '{node}' LABEL 'l' DIRECTION out" + )) + .await + .expect("neighbors"); + let mut out = Vec::new(); + for row in rows { + let parsed: Vec = serde_json::from_str(&row).unwrap_or_default(); + for entry in parsed { + if let Some(n) = entry.get("node").and_then(|v| v.as_str()) { + out.push(n.to_string()); + } + } + } + out +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn txn_block_update_of_edge_target_moves_the_graph_edge() { + let server = TestServer::start().await; + server.exec("CREATE COLLECTION ge").await.unwrap(); + server + .exec("INSERT INTO ge { id: 'e1', _from: 'a', _to: 'b', _type: 'l' }") + .await + .unwrap(); + assert_eq!( + neighbors_of(&server, "ge", "a").await, + vec!["b".to_string()] + ); + + server.exec("BEGIN").await.unwrap(); + server + .exec("UPDATE ge SET _to = 'c' WHERE id = 'e1'") + .await + .unwrap(); + server.exec("COMMIT").await.expect("COMMIT must succeed"); + + let after = neighbors_of(&server, "ge", "a").await; + assert_eq!( + after, + vec!["c".to_string()], + "the traversal follows the moved edge, not the old one" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn bitemporal_predicate_delete_commits_through_calvin() { + let server = TestServer::start().await; + for sql in [ + "CREATE COLLECTION bt (id STRING PRIMARY KEY, region STRING) \ + WITH (engine='document_schemaless', bitemporal=true)", + "INSERT INTO bt (id, region) VALUES ('r0', 'us')", + "INSERT INTO bt (id, region) VALUES ('r1', 'eu')", + "INSERT INTO bt (id, region) VALUES ('r2', 'eu')", + // An edge makes the collection edge-bearing, so a delete carries + // node guards on the rows' key vShards and COMMIT takes Calvin. + "GRAPH INSERT EDGE IN 'bt' FROM 'r0' TO 'r1' TYPE 'l'", + ] { + server + .exec(sql) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")); + } + + server.exec("BEGIN").await.unwrap(); + server + .exec("DELETE FROM bt WHERE region = 'eu'") + .await + .unwrap(); + server.exec("COMMIT").await.expect("COMMIT must succeed"); + + let mut ids: Vec = server + .query_rows("SELECT id FROM bt") + .await + .expect("select") + .into_iter() + .filter_map(|row| row.into_iter().next()) + .collect(); + ids.sort(); + assert_eq!(ids, vec!["r0".to_string()]); + assert!( + neighbors_of(&server, "bt", "r0").await.is_empty(), + "the deleted node's incident edge goes with it" + ); +} diff --git a/nodedb/tests/wire/cases/calvin_txn_predicate_writes.rs b/nodedb/tests/wire/cases/calvin_txn_predicate_writes.rs new file mode 100644 index 000000000..02f3bc97d --- /dev/null +++ b/nodedb/tests/wire/cases/calvin_txn_predicate_writes.rs @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A predicate `UPDATE` or `DELETE` inside a transaction commits through +//! Calvin. +//! +//! A delete on an edge-bearing collection carries one node guard per deleted +//! row, homed on the row's key vShard. The COMMIT then spans two vShards and +//! takes the Calvin path. The predicate write carries the rows its statement +//! read, so the Calvin stage writes exactly those rows. A statement inside a +//! transaction block, a trigger body and a procedure body all buffer their +//! writes the same way. + +use crate::harness::TestServer; + +/// Four rows and one edge `d0 -> d1` in the edge-bearing collection `g`. +async fn seed_edge_collection(server: &TestServer) { + for sql in [ + "CREATE COLLECTION g", + "UPSERT INTO g { id: 'd0', title: 't0' }", + "UPSERT INTO g { id: 'd1', title: 't1' }", + "UPSERT INTO g { id: 'd2', title: 't2' }", + "UPSERT INTO g { id: 'd3', title: 't3' }", + "GRAPH INSERT EDGE IN 'g' FROM 'd0' TO 'd1' TYPE 'l'", + ] { + server + .exec(sql) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")); + } +} + +/// The ids of the rows of `g` the predicate `filter` matches. +async fn ids_where(server: &TestServer, filter: &str) -> Vec { + let mut ids: Vec = server + .query_rows(&format!("SELECT id FROM g WHERE {filter}")) + .await + .expect("select") + .into_iter() + .filter_map(|row| row.into_iter().next()) + .collect(); + ids.sort(); + ids +} + +/// The destination nodes of `node`'s outgoing `l` edges in `g`. +async fn neighbors_of(server: &TestServer, node: &str) -> Vec { + let rows = server + .query_text(&format!( + "GRAPH NEIGHBORS IN 'g' OF '{node}' LABEL 'l' DIRECTION out" + )) + .await + .expect("neighbors"); + let mut out = Vec::new(); + for row in rows { + let parsed: Vec = serde_json::from_str(&row).unwrap_or_default(); + for entry in parsed { + if let Some(n) = entry.get("node").and_then(|v| v.as_str()) { + out.push(n.to_string()); + } + } + } + out +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn txn_block_delete_on_edge_bearing_collection_commits() { + let server = TestServer::start().await; + seed_edge_collection(&server).await; + + server.exec("BEGIN").await.unwrap(); + server.exec("DELETE FROM g WHERE id = 'd3'").await.unwrap(); + server + .exec("COMMIT") + .await + .expect("a Calvin COMMIT of a buffered predicate delete must succeed"); + + assert!(ids_where(&server, "id = 'd3'").await.is_empty()); + assert_eq!(neighbors_of(&server, "d0").await, vec!["d1".to_string()]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn txn_block_update_and_delete_on_edge_bearing_collection_commits() { + let server = TestServer::start().await; + seed_edge_collection(&server).await; + + server.exec("BEGIN").await.unwrap(); + server + .exec("UPDATE g SET title = 'x' WHERE id = 'd2'") + .await + .unwrap(); + server.exec("DELETE FROM g WHERE id = 'd3'").await.unwrap(); + server.exec("COMMIT").await.expect("COMMIT must succeed"); + + assert_eq!( + ids_where(&server, "title = 'x'").await, + vec!["d2".to_string()] + ); + assert!(ids_where(&server, "id = 'd3'").await.is_empty()); +} + +/// The delete's prediction is read as the transaction sees its rows, so a +/// row the same block inserted is one it deletes. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn txn_block_delete_matches_its_own_earlier_insert() { + let server = TestServer::start().await; + seed_edge_collection(&server).await; + + server.exec("BEGIN").await.unwrap(); + server + .exec("UPSERT INTO g { id: 'd9', title: 'mine' }") + .await + .unwrap(); + server + .exec("DELETE FROM g WHERE title = 'mine'") + .await + .unwrap(); + server.exec("COMMIT").await.expect("COMMIT must succeed"); + + assert!(ids_where(&server, "title = 'mine'").await.is_empty()); +} + +/// A SYNC AFTER body joins its statement's implicit transaction. Its +/// predicate delete on an edge-bearing collection commits with the insert. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn trigger_body_predicate_delete_commits_with_its_insert() { + let server = TestServer::start().await; + seed_edge_collection(&server).await; + for ddl in [ + "CREATE COLLECTION orders", + "CREATE SYNC TRIGGER drop_title AFTER INSERT ON orders FOR EACH ROW \ + BEGIN DELETE FROM g WHERE title = NEW.title; END", + ] { + server + .exec(ddl) + .await + .unwrap_or_else(|e| panic!("{ddl}: {e}")); + } + + server + .exec("INSERT INTO orders { id: 'o1', title: 't3' }") + .await + .expect("the insert and its body's predicate delete must commit"); + + assert!(ids_where(&server, "id = 'd3'").await.is_empty()); + let orders = server + .query_rows("SELECT id FROM orders WHERE id = 'o1'") + .await + .expect("select orders"); + assert_eq!(orders.len(), 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn procedure_body_predicate_delete_on_edge_bearing_collection_commits() { + let server = TestServer::start().await; + seed_edge_collection(&server).await; + server + .exec( + "CREATE PROCEDURE prune() AS \ + BEGIN \ + UPDATE g SET title = 'x' WHERE id = 'd2'; \ + DELETE FROM g WHERE id = 'd3'; \ + END", + ) + .await + .expect("CREATE PROCEDURE"); + + server.exec("CALL prune()").await.expect("CALL must commit"); + + assert_eq!( + ids_where(&server, "title = 'x'").await, + vec!["d2".to_string()] + ); + assert!(ids_where(&server, "id = 'd3'").await.is_empty()); +} + +/// A row that joins the predicate after the statement read it moves the +/// prediction. COMMIT refuses the stale set with a serialization conflict +/// and writes nothing. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn txn_block_commit_after_a_concurrent_match_change_aborts_40001() { + const PASSWORD: &str = "calvin-txn-predicate-7"; + let server = TestServer::start().await; + seed_edge_collection(&server).await; + server + .exec(&format!("CREATE USER writer PASSWORD '{PASSWORD}'")) + .await + .expect("CREATE USER"); + let (writer, handle) = server + .connect_as("writer", PASSWORD) + .await + .expect("second connection"); + + server.exec("BEGIN").await.unwrap(); + server + .exec("DELETE FROM g WHERE title = 't3'") + .await + .unwrap(); + writer + .simple_query("UPSERT INTO g { id: 'n1', title: 't3' }") + .await + .expect("the concurrent write commits"); + drop(writer); + handle.abort(); + + let commit = server.client.simple_query("COMMIT").await; + let error = commit.expect_err("a stale prediction must abort the COMMIT"); + let code = error.as_db_error().map(|e| e.code().code().to_string()); + assert_eq!(code.as_deref(), Some("40001"), "{error:?}"); + + let mut survivors = ids_where(&server, "title = 't3'").await; + survivors.sort(); + assert_eq!(survivors, vec!["d3".to_string(), "n1".to_string()]); +} diff --git a/nodedb/tests/wire/cases/graph_rag_fusion_naming.rs b/nodedb/tests/wire/cases/graph_rag_fusion_naming.rs new file mode 100644 index 000000000..2e63c32c0 --- /dev/null +++ b/nodedb/tests/wire/cases/graph_rag_fusion_naming.rs @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `GRAPH RAG FUSION` names every row by its primary key. +//! +//! A row in no edge has no graph node name. The fusion keys it by its +//! surrogate, and the response boundary names it through the resolver every +//! search response uses. A client then joins each `node_id` to its document. +//! Naming runs after fusion: a key equal to another row's surrogate decimal +//! never fuses with that row. + +use crate::harness::TestServer; + +/// The `results` array of a fusion answer. +async fn fusion_results(server: &TestServer, sql: &str) -> Vec { + let blob = server + .query_text(sql) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")) + .join(""); + let parsed: serde_json::Value = + serde_json::from_str(&blob).unwrap_or_else(|e| panic!("not JSON: {e}: {blob}")); + parsed + .get("results") + .and_then(|r| r.as_array()) + .cloned() + .unwrap_or_else(|| panic!("no results array: {blob}")) +} + +/// Each result's `node_id`, in rank order. +fn node_ids(results: &[serde_json::Value]) -> Vec { + results + .iter() + .map(|r| { + r.get("node_id") + .and_then(|n| n.as_str()) + .unwrap_or_else(|| panic!("a result carries no node_id: {r}")) + .to_string() + }) + .collect() +} + +/// The single result named `node_id`. +fn result_named<'a>(results: &'a [serde_json::Value], node_id: &str) -> &'a serde_json::Value { + let named: Vec<&serde_json::Value> = results + .iter() + .filter(|r| r.get("node_id").and_then(|n| n.as_str()) == Some(node_id)) + .collect(); + assert_eq!( + named.len(), + 1, + "exactly one result must be named {node_id:?}; got {results:?}" + ); + named[0] +} + +/// Every `node_id` names a row the client can read back by its key. +async fn assert_every_node_id_joins(server: &TestServer, collection: &str, ids: &[String]) { + for id in ids { + let rows = server + .query_text(&format!("SELECT id FROM {collection} WHERE id = '{id}'")) + .await + .unwrap_or_else(|e| panic!("select {id}: {e}")); + assert_eq!( + rows.len(), + 1, + "node_id {id:?} must join to exactly one row of {collection}; got {rows:?}" + ); + } +} + +async fn vector_collection(server: &TestServer, name: &str) { + server + .exec(&format!("CREATE COLLECTION {name}")) + .await + .unwrap(); + server + .exec(&format!( + "CREATE VECTOR INDEX idx_{name}_emb ON {name} METRIC cosine DIM 2" + )) + .await + .unwrap(); +} + +async fn text_collection(server: &TestServer, name: &str) { + vector_collection(server, name).await; + server + .exec(&format!( + "CREATE SEARCH INDEX idx_{name}_fts ON {name} FIELDS body ANALYZER 'standard'" + )) + .await + .unwrap(); +} + +/// The vector-nearest row is in no edge. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn vector_hit_in_no_edge_returns_its_primary_key() { + let server = TestServer::start().await; + vector_collection(&server, "ragn_vec").await; + for (id, v) in [ + ("lonely", "1.0, 0.0"), + ("a", "0.0, 1.0"), + ("b", "-1.0, 0.0"), + ] { + server + .exec(&format!( + "INSERT INTO ragn_vec (id, embedding) VALUES ('{id}', ARRAY[{v}])" + )) + .await + .unwrap(); + } + server + .exec("GRAPH INSERT EDGE IN 'ragn_vec' FROM 'a' TO 'b' TYPE 'hop'") + .await + .unwrap(); + + let results = fusion_results( + &server, + "GRAPH RAG FUSION ON ragn_vec QUERY ARRAY[1.0, 0.0] VECTOR_FIELD 'embedding' \ + VECTOR_TOP_K 2 EXPANSION_DEPTH 1 EDGE_LABEL 'hop' FINAL_TOP_K 10", + ) + .await; + let ids = node_ids(&results); + assert!( + ids.contains(&"lonely".to_string()), + "the hit in no edge must answer its key; got {ids:?}" + ); + assert!( + results.iter().all(|r| r.get("surrogate").is_none()), + "the naming marker never reaches a client: {results:?}" + ); + assert_every_node_id_joins(&server, "ragn_vec", &ids).await; +} + +/// A text-only hit in no edge, three sources. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn text_hit_in_no_edge_returns_its_primary_key() { + let server = TestServer::start().await; + text_collection(&server, "ragn_text").await; + server + .exec( + "INSERT INTO ragn_text (id, body, embedding) \ + VALUES ('near', 'plain prose', ARRAY[1.0, 0.0])", + ) + .await + .unwrap(); + // `wordy` is far from the query vector and the only `entangled` match. + server + .exec( + "INSERT INTO ragn_text (id, body, embedding) \ + VALUES ('wordy', 'entangled photons', ARRAY[-1.0, 0.0])", + ) + .await + .unwrap(); + + let results = fusion_results( + &server, + "GRAPH RAG FUSION ON ragn_text QUERY ARRAY[1.0, 0.0] VECTOR_FIELD 'embedding' \ + VECTOR_TOP_K 1 BM25 'entangled' ON 'body' EXPANSION_DEPTH 1 FINAL_TOP_K 10 \ + RRF_K (60.0, 60.0, 60.0)", + ) + .await; + let ids = node_ids(&results); + assert!( + ids.contains(&"wordy".to_string()), + "the text-only hit must answer its key; got {ids:?}" + ); + assert_every_node_id_joins(&server, "ragn_text", &ids).await; +} + +/// Both legs hit one row in no edge: one named entry with both contributions. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn vector_and_text_hit_in_no_edge_returns_one_named_row() { + let server = TestServer::start().await; + text_collection(&server, "ragn_both").await; + server + .exec( + "INSERT INTO ragn_both (id, body, embedding) \ + VALUES ('both', 'entangled photons', ARRAY[1.0, 0.0])", + ) + .await + .unwrap(); + server + .exec( + "INSERT INTO ragn_both (id, body, embedding) \ + VALUES ('other', 'plain prose', ARRAY[0.9, 0.1])", + ) + .await + .unwrap(); + + // Text rank 0 alone is 0.5 under text_k = 1; both legs pass 0.5. + let results = fusion_results( + &server, + "GRAPH RAG FUSION ON ragn_both QUERY ARRAY[1.0, 0.0] VECTOR_FIELD 'embedding' \ + VECTOR_TOP_K 5 BM25 'entangled' ON 'body' EXPANSION_DEPTH 1 FINAL_TOP_K 10 \ + RRF_K (60.0, 1.0, 60.0)", + ) + .await; + let both = result_named(&results, "both"); + let score = both + .get("rrf_score") + .and_then(serde_json::Value::as_f64) + .unwrap_or_default(); + assert!(score > 0.5, "both legs must fuse onto `both`: {results:?}"); +} + +/// A text hit on a graph node fuses with that node's vector and graph entry. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn text_hit_on_a_graph_node_fuses_with_its_node() { + let server = TestServer::start().await; + text_collection(&server, "ragn_node").await; + server + .exec( + "INSERT INTO ragn_node (id, body, embedding) \ + VALUES ('a', 'entangled photons', ARRAY[1.0, 0.0])", + ) + .await + .unwrap(); + server + .exec( + "INSERT INTO ragn_node (id, body, embedding) \ + VALUES ('b', 'plain prose', ARRAY[-1.0, 0.0])", + ) + .await + .unwrap(); + server + .exec("GRAPH INSERT EDGE IN 'ragn_node' FROM 'a' TO 'b' TYPE 'hop'") + .await + .unwrap(); + + let results = fusion_results( + &server, + "GRAPH RAG FUSION ON ragn_node QUERY ARRAY[1.0, 0.0] VECTOR_FIELD 'embedding' \ + VECTOR_TOP_K 1 BM25 'entangled' ON 'body' EXPANSION_DEPTH 1 EDGE_LABEL 'hop' \ + FINAL_TOP_K 10 RRF_K (60.0, 1.0, 60.0)", + ) + .await; + let a = result_named(&results, "a"); + assert_eq!( + a.get("hop_distance").and_then(serde_json::Value::as_u64), + Some(0), + "the fused entry is the graph node itself: {results:?}" + ); + let score = a + .get("rrf_score") + .and_then(serde_json::Value::as_f64) + .unwrap_or_default(); + assert!( + score > 0.5, + "the text leg must fuse with the node's entry: {results:?}" + ); +} + +/// A strict collection's row in no edge answers its declared key. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn strict_collection_hit_returns_its_declared_key() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION ragn_strict (code TEXT PRIMARY KEY, embedding VECTOR(2)) \ + WITH (engine = 'document_strict')", + ) + .await + .unwrap(); + server + .exec("CREATE VECTOR INDEX idx_ragn_strict ON ragn_strict (embedding) METRIC cosine DIM 2") + .await + .unwrap(); + for (code, v) in [("lonely", "[1.0, 0.0]"), ("far", "[-1.0, 0.0]")] { + server + .exec(&format!( + "INSERT INTO ragn_strict (code, embedding) VALUES ('{code}', '{v}')" + )) + .await + .unwrap(); + } + + let results = fusion_results( + &server, + "GRAPH RAG FUSION ON ragn_strict QUERY ARRAY[1.0, 0.0] VECTOR_FIELD 'embedding' \ + VECTOR_TOP_K 2 EXPANSION_DEPTH 1 FINAL_TOP_K 10", + ) + .await; + let ids = node_ids(&results); + assert!( + ids.contains(&"lonely".to_string()) && ids.contains(&"far".to_string()), + "a strict row answers its declared key; got {ids:?}" + ); +} + +/// A row inserted with no id answers the decimal identity `SELECT id` reads. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn minted_row_returns_its_decimal_identity() { + let server = TestServer::start().await; + vector_collection(&server, "ragn_minted").await; + server + .exec("INSERT INTO ragn_minted (embedding) VALUES (ARRAY[1.0, 0.0])") + .await + .unwrap(); + let ids = server + .query_text("SELECT id FROM ragn_minted") + .await + .expect("select id"); + assert_eq!(ids.len(), 1, "one minted row; got {ids:?}"); + + let results = fusion_results( + &server, + "GRAPH RAG FUSION ON ragn_minted QUERY ARRAY[1.0, 0.0] VECTOR_FIELD 'embedding' \ + VECTOR_TOP_K 1 EXPANSION_DEPTH 1 FINAL_TOP_K 10", + ) + .await; + assert_eq!( + node_ids(&results), + ids, + "the minted row answers the identity SELECT id reads" + ); +} + +/// A key equal to another row's surrogate decimal stays a separate entry. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn decimal_key_stays_apart_from_an_equal_surrogate() { + let server = TestServer::start().await; + vector_collection(&server, "ragn_decimal").await; + server + .exec("INSERT INTO ragn_decimal (id, embedding) VALUES ('x', ARRAY[1.0, 0.0])") + .await + .unwrap(); + let surrogate = server + .query_text( + "SELECT s._surrogate FROM \ + (SELECT * FROM ragn_decimal \ + ORDER BY vector_distance(embedding, ARRAY[1.0, 0.0]) LIMIT 1) s", + ) + .await + .expect("read x's surrogate") + .into_iter() + .next() + .expect("x has a surrogate"); + // Row `` is a graph node one hop from `y`; `x` is in no edge. + server + .exec(&format!( + "INSERT INTO ragn_decimal (id, embedding) VALUES ('{surrogate}', ARRAY[-1.0, 0.0])" + )) + .await + .unwrap(); + server + .exec("INSERT INTO ragn_decimal (id, embedding) VALUES ('y', ARRAY[0.0, 1.0])") + .await + .unwrap(); + server + .exec(&format!( + "GRAPH INSERT EDGE IN 'ragn_decimal' FROM 'y' TO '{surrogate}' TYPE 'hop'" + )) + .await + .unwrap(); + + let results = fusion_results( + &server, + "GRAPH RAG FUSION ON ragn_decimal QUERY ARRAY[1.0, 0.0] VECTOR_FIELD 'embedding' \ + VECTOR_TOP_K 2 EXPANSION_DEPTH 1 EDGE_LABEL 'hop' FINAL_TOP_K 10", + ) + .await; + let x = result_named(&results, "x"); + assert!( + x.get("hop_distance").is_none_or(serde_json::Value::is_null), + "x is in no edge: {results:?}" + ); + let node = result_named(&results, &surrogate); + assert_eq!( + node.get("hop_distance").and_then(serde_json::Value::as_u64), + Some(1), + "the decimal-keyed node is reached by expansion: {results:?}" + ); + assert!( + node.get("vector_rank") + .is_none_or(serde_json::Value::is_null), + "the decimal-keyed node never takes x's vector rank: {results:?}" + ); +} diff --git a/nodedb/tests/wire/cases/materialized_sum_txn_predicate_writes.rs b/nodedb/tests/wire/cases/materialized_sum_txn_predicate_writes.rs new file mode 100644 index 000000000..14d95c1f2 --- /dev/null +++ b/nodedb/tests/wire/cases/materialized_sum_txn_predicate_writes.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A predicate `UPDATE` or `DELETE` inside a transaction block, on the source +//! of a materialized sum whose target lives on another vShard. +//! +//! The block's COMMIT spans the source and the target vShard, so it runs as a +//! Calvin transaction. Each predicate write carries the rows its statement +//! read, and the sum targets those rows resolve to. COMMIT applies the source +//! rows and the balance together. + +use super::materialized_sum_cross_shard_expanders::{balance, sum_fixture}; +use crate::harness::TestServer; + +async fn exec(server: &TestServer, sql: &str) { + server + .exec(sql) + .await + .unwrap_or_else(|e| panic!("run {sql}: {e}")); +} + +/// `UPDATE ... WHERE` on a key moves the balance by the amount's difference. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_block_update_by_key_moves_a_cross_shard_sum() { + let server = sum_fixture().await; + exec( + &server, + "BEGIN; UPDATE ptv_post SET amount = '20' WHERE id = 'e0'; COMMIT", + ) + .await; + assert_eq!(balance(&server, "acc").await, "25", "e0 at 20, e1 at 5"); +} + +/// `UPDATE ... WHERE` on a non-key column moves the balance of every row it +/// matches. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_block_update_by_predicate_moves_a_cross_shard_sum() { + let server = sum_fixture().await; + exec( + &server, + "BEGIN; UPDATE ptv_post SET amount = '1' WHERE account_id = 'acc'; COMMIT", + ) + .await; + assert_eq!(balance(&server, "acc").await, "2", "e0 and e1 at 1"); +} + +/// `DELETE ... WHERE` takes the deleted posting off the balance. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_block_delete_takes_its_posting_off_a_cross_shard_sum() { + let server = sum_fixture().await; + exec( + &server, + "BEGIN; DELETE FROM ptv_post WHERE id = 'e1'; COMMIT", + ) + .await; + assert_eq!(balance(&server, "acc").await, "10", "e1 deleted"); +} + +/// A posting the block inserts and then moves by predicate lands on the +/// balance once, at its final amount. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_block_update_matches_the_blocks_own_insert() { + let server = sum_fixture().await; + exec( + &server, + "BEGIN; \ + INSERT INTO ptv_post (id, account_id, amount) VALUES ('e2', 'acc2', '4'); \ + UPDATE ptv_post SET amount = '9' WHERE account_id = 'acc2'; \ + COMMIT", + ) + .await; + assert_eq!( + balance(&server, "acc2").await, + "9", + "e2 inserted, then at 9" + ); + assert_eq!(balance(&server, "acc").await, "15", "acc untouched"); +} diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index fce34324f..2787346bc 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -8,6 +8,7 @@ mod aggregate_non_finite_float; mod aggregate_over_join; mod aggregate_routing_multicore; mod array_agg_group_types; +mod array_agg_relation; mod array_copy_from; mod array_dml_in_bodies; mod array_georef; @@ -22,9 +23,13 @@ mod bitemporal_array_sql; mod bitemporal_asof_scan_parity; mod bitemporal_delete_index_tombstone; mod bitemporal_document_txn_restart; +mod bitemporal_index_convert; +mod bitemporal_predicate_writes; mod bulk_dml_event_emission; mod calvin_multi_part_node_delete; mod calvin_sql_routing; +mod calvin_txn_edge_update; +mod calvin_txn_predicate_writes; mod clone_alter_materialize; mod clone_as_of_past_time; mod clone_carries_collection_metadata; @@ -121,6 +126,7 @@ mod graph_dsl_label_sets; mod graph_edge_predicate; mod graph_match_authorization; mod graph_rag_fusion; +mod graph_rag_fusion_naming; mod graph_timeseries_rls_probe; mod graph_traversal_authorization; mod graph_traverse_absent_start; @@ -149,6 +155,7 @@ mod kv_write_row_level_security_atomics; mod materialized_sum_cross_shard_expanders; mod materialized_sum_cross_shard_http; mod materialized_sum_txn_command_tag; +mod materialized_sum_txn_predicate_writes; mod materialized_sum_txn_preimage; mod merge_insert_atomicity; mod merge_insert_cross_engine_visibility; diff --git a/nodedb/tests/wire/cases/trigger_e2e.rs b/nodedb/tests/wire/cases/trigger_e2e.rs index d3445dddb..6342f258d 100644 --- a/nodedb/tests/wire/cases/trigger_e2e.rs +++ b/nodedb/tests/wire/cases/trigger_e2e.rs @@ -581,13 +581,11 @@ async fn publish_to_inside_trigger_body_delivers_message() { .await .unwrap(); - // Give the event plane a moment to dispatch the AFTER trigger. - tokio::time::sleep(Duration::from_millis(500)).await; - - let rows = server - .query_text("SELECT * FROM TOPIC order_events CONSUMER GROUP processors LIMIT 10") - .await - .expect("SELECT FROM TOPIC should succeed"); + let rows = published_rows( + &server, + "SELECT * FROM TOPIC order_events CONSUMER GROUP processors LIMIT 10", + ) + .await; assert!( !rows.is_empty(), @@ -628,9 +626,7 @@ async fn publish_to_payload_unescapes_doubled_quotes() { .await .unwrap(); - tokio::time::sleep(Duration::from_millis(500)).await; - - let combined = topic_rows_joined( + let combined = published_rows( &server, "SELECT * FROM TOPIC quote_events CONSUMER GROUP qg LIMIT 10", ) @@ -645,6 +641,29 @@ async fn publish_to_payload_unescapes_doubled_quotes() { ); } +/// How long an AFTER trigger's `PUBLISH TO` may take to reach its topic. +const PUBLISH_DEADLINE: Duration = Duration::from_secs(15); +/// The interval between topic reads while a published message is awaited. +const PUBLISH_POLL: Duration = Duration::from_millis(50); + +/// The joined rows `sql` reads once the AFTER trigger's message reaches the +/// topic. The trigger runs on the event plane after the INSERT returns, so +/// the read repeats until a row arrives. Panics past [`PUBLISH_DEADLINE`]. +async fn published_rows(server: &TestServer, sql: &str) -> String { + let start = std::time::Instant::now(); + loop { + let rows = topic_rows_joined(server, sql).await; + if !rows.is_empty() { + return rows; + } + assert!( + start.elapsed() < PUBLISH_DEADLINE, + "no message reached the topic within {PUBLISH_DEADLINE:?}: {sql}" + ); + tokio::time::sleep(PUBLISH_POLL).await; + } +} + async fn topic_rows_joined(server: &TestServer, sql: &str) -> String { let msgs = server.client.simple_query(sql).await.unwrap(); let mut out = String::new(); @@ -693,12 +712,11 @@ async fn publish_to_empty_quoted_payload_delivers() { .await .unwrap(); - tokio::time::sleep(Duration::from_millis(500)).await; - - let rows = server - .query_text("SELECT * FROM TOPIC empty_events CONSUMER GROUP eg LIMIT 10") - .await - .unwrap(); + let rows = published_rows( + &server, + "SELECT * FROM TOPIC empty_events CONSUMER GROUP eg LIMIT 10", + ) + .await; assert!( !rows.is_empty(), @@ -740,9 +758,7 @@ async fn publish_to_new_field_substitution_delivers_value() { .await .unwrap(); - tokio::time::sleep(Duration::from_millis(500)).await; - - let combined = topic_rows_joined( + let combined = published_rows( &server, "SELECT * FROM TOPIC user_events CONSUMER GROUP ug LIMIT 10", )