diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 7501202c1..d95fa3a06 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -1,4 +1,4 @@ -# Seven jobs on a nightly toolchain — only worth running when a change can +# Eight jobs on a nightly toolchain — only worth running when a change can # reach a decoder. The weekly cron runs unconditionally, so a too-narrow path # filter costs coverage for a week, not forever. @@ -7,10 +7,11 @@ name: ASan fuzz + 32-bit corpus/property on: pull_request: branches: [main] - # The crates the six targets reach. + # The crates the seven targets reach. paths: - fuzz/** - nodedb-codec/** + - nodedb-raster/** - nodedb-columnar/** - nodedb-strict/** - nodedb-wal/** @@ -49,6 +50,8 @@ jobs: dictionary: fuzz/dictionaries/binary.dict - name: sql_preprocess dictionary: fuzz/dictionaries/sql.dict + - name: geotiff_decoder + dictionary: fuzz/dictionaries/binary.dict steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 # A pinned nightly keeps the build cache key stable across days. Move @@ -63,7 +66,7 @@ jobs: with: workspaces: fuzz -> target key: ${{ matrix.target.name }} - # All six matrix jobs install the same pinned binary; build it once. + # All seven matrix jobs install the same pinned binary; build it once. - name: Cache cargo-fuzz id: cache-cargo-fuzz uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 diff --git a/Cargo.lock b/Cargo.lock index 3c6878b1e..ed330df75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3974,6 +3974,7 @@ dependencies = [ "nodedb-physical", "nodedb-query", "nodedb-raft", + "nodedb-raster", "nodedb-spatial", "nodedb-sql", "nodedb-strict", @@ -4015,6 +4016,7 @@ dependencies = [ "subtle", "tempfile", "thiserror 2.0.21", + "tiff", "tikv-jemalloc-ctl", "tikv-jemallocator", "tokio", @@ -4042,6 +4044,7 @@ dependencies = [ "crc32c", "loro", "nodedb-codec", + "nodedb-spatial", "nodedb-types", "nodedb-wal", "rustc-hash", @@ -4340,6 +4343,15 @@ dependencies = [ "zerompk", ] +[[package]] +name = "nodedb-raster" +version = "0.7.0-dev" +dependencies = [ + "nodedb-types", + "thiserror 2.0.21", + "tiff", +] + [[package]] name = "nodedb-spatial" version = "0.7.0-dev" @@ -5386,6 +5398,12 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quick-xml" version = "0.41.0" @@ -6111,7 +6129,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" dependencies = [ "fnv", - "quick-error", + "quick-error 1.2.3", "tempfile", "wait-timeout", ] @@ -6823,6 +6841,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "flate2", + "half", + "quick-error 2.0.1", + "weezl", +] + [[package]] name = "tikv-jemalloc-ctl" version = "0.7.0" @@ -7798,6 +7828,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "whatlang" version = "0.18.0" diff --git a/Cargo.toml b/Cargo.toml index 8dcf54aa3..0cec15a63 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "nodedb-strict", "nodedb-columnar", "nodedb-array", + "nodedb-raster", "nodedb-sql", "nodedb-physical", "nodedb-test-support", @@ -58,6 +59,7 @@ nodedb-fts = { path = "nodedb-fts", version = "0.7.0-dev" } nodedb-strict = { path = "nodedb-strict", version = "0.7.0-dev" } nodedb-columnar = { path = "nodedb-columnar", version = "0.7.0-dev" } nodedb-array = { path = "nodedb-array", version = "0.7.0-dev" } +nodedb-raster = { path = "nodedb-raster", version = "0.7.0-dev" } nodedb-sql = { path = "nodedb-sql", version = "0.7.0-dev" } nodedb-physical = { path = "nodedb-physical", version = "0.7.0-dev" } nodedb-test-support = { path = "nodedb-test-support" } @@ -94,6 +96,10 @@ libc = "0.2" # Checksums crc32c = "0.6" +# GeoTIFF decoding with the deflate and LZW codecs only. +# The raster reader refuses JPEG, fax and WebP compression. +tiff = { version = "0.11.3", default-features = false, features = ["deflate", "lzw"] } + # Crypto # Pinned to the digest 0.10 generation: `rsa` (JWT RS256) has no release on # digest 0.11, so sha2 0.11 / hmac 0.13 / aes-gcm 0.11 / p256 0.14 cannot be diff --git a/docs/query-language.md b/docs/query-language.md index 69118e8b4..afa3c3edb 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -1187,12 +1187,14 @@ COPY users TO '/exports/users.ndjson'; COPY (SELECT id, name FROM users WHERE status = 'active') TO '/exports/active.csv' WITH (FORMAT csv); ``` +- A server-path `COPY FROM` or `COPY TO` reads or writes as the server's OS user, so it needs superuser. Any other user is `42501`. - A path must be absolute and must not contain `..`. A relative path is `42601`. A path with `..` is `42501`. +- `COPY FROM` reads regular files only. A FIFO, a device, or a directory is `58030`, also through a symlink. - `FORMAT` is `ndjson` (or `jsonl`), `json`, or `csv`. Without `FORMAT`, the extension decides: `.ndjson`, `.jsonl`, `.json`, `.csv`. `DELIMITER` is one character. `HEADER` defaults to `true`. - `COPY FROM STDIN` is refused with `0A000`. Use the server-path form. - A collection on the `timeseries` or `spatial` engine refuses `COPY FROM` with `0A000`. - A file over 16 GiB is `54000`. -- `COPY TO` writes `.tmp` and renames it to ``. It is not transactional. +- `COPY TO` writes `.tmp` and renames it to ``. `.tmp` must not exist, else `58030`. It is not transactional. - `COPY FROM ''` loads CSV only. Use `WITH (FORMAT csv)`. ### Array bulk load diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index d27e9d03d..0b66b90ad 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aead" version = "0.5.2" @@ -574,6 +580,15 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-deque" version = "0.8.8" @@ -760,6 +775,17 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "float_eq" version = "1.0.1" @@ -1367,6 +1393,16 @@ dependencies = [ "libc", ] +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "multiversion_no_op" version = "1.0.0" @@ -1497,6 +1533,7 @@ dependencies = [ "nodedb-codec", "nodedb-columnar", "nodedb-mem", + "nodedb-raster", "nodedb-sql", "nodedb-strict", "nodedb-types", @@ -1532,6 +1569,15 @@ dependencies = [ "zerompk", ] +[[package]] +name = "nodedb-raster" +version = "0.7.0-dev" +dependencies = [ + "nodedb-types", + "thiserror", + "tiff", +] + [[package]] name = "nodedb-spatial" version = "0.7.0-dev" @@ -1823,6 +1869,12 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quote" version = "1.0.47" @@ -2245,6 +2297,12 @@ dependencies = [ "wide", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "simdutf8" version = "0.1.5" @@ -2436,6 +2494,18 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "flate2", + "half", + "quick-error", + "weezl", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -2677,6 +2747,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "whatlang" version = "0.18.0" @@ -2924,6 +3000,12 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index c8b97e530..bd6e25a09 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -17,6 +17,7 @@ tempfile = "3" nodedb-codec = { path = "../nodedb-codec" } nodedb-columnar = { path = "../nodedb-columnar" } nodedb-mem = { path = "../nodedb-mem" } +nodedb-raster = { path = "../nodedb-raster" } nodedb-strict = { path = "../nodedb-strict" } nodedb-types = { path = "../nodedb-types" } nodedb-wal = { path = "../nodedb-wal" } @@ -62,6 +63,14 @@ doc = false bench = false required-features = ["fuzzing"] +[[bin]] +name = "geotiff_decoder" +path = "fuzz_targets/geotiff_decoder.rs" +test = false +doc = false +bench = false +required-features = ["fuzzing"] + [[bin]] name = "sql_preprocess" path = "fuzz_targets/sql_preprocess.rs" diff --git a/fuzz/corpus/geotiff_decoder/strip-f32.tif b/fuzz/corpus/geotiff_decoder/strip-f32.tif new file mode 100644 index 000000000..8fd48b34d Binary files /dev/null and b/fuzz/corpus/geotiff_decoder/strip-f32.tif differ diff --git a/fuzz/corpus/geotiff_decoder/tiled-deflate-u16x3.tif b/fuzz/corpus/geotiff_decoder/tiled-deflate-u16x3.tif new file mode 100644 index 000000000..b1fb4e422 Binary files /dev/null and b/fuzz/corpus/geotiff_decoder/tiled-deflate-u16x3.tif differ diff --git a/fuzz/corpus/geotiff_decoder/truncated.tif b/fuzz/corpus/geotiff_decoder/truncated.tif new file mode 100644 index 000000000..87796cc4a Binary files /dev/null and b/fuzz/corpus/geotiff_decoder/truncated.tif differ diff --git a/fuzz/fuzz_targets/geotiff_decoder.rs b/fuzz/fuzz_targets/geotiff_decoder.rs new file mode 100644 index 000000000..9b0b87a27 --- /dev/null +++ b/fuzz/fuzz_targets/geotiff_decoder.rs @@ -0,0 +1,5 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| nodedb_fuzz::targets::geotiff_decoder::run(data)); diff --git a/fuzz/src/targets/geotiff_decoder.rs b/fuzz/src/targets/geotiff_decoder.rs new file mode 100644 index 000000000..89ef06017 --- /dev/null +++ b/fuzz/src/targets/geotiff_decoder.rs @@ -0,0 +1,34 @@ +//! GeoTIFF open and block decode over arbitrary bytes. +//! +//! Every outcome must be `Ok` or a typed `RasterError`. A decoded block +//! must hold exactly one window of samples per band. + +use std::io::Cursor; + +use nodedb_raster::{GeoTiffReader, RasterLimits}; + +/// Per-block decode cap. Small enough that a hostile header cannot make +/// one block a large allocation. +const BLOCK_BYTES: usize = 4 * 1024 * 1024; +/// Blocks decoded per input, so a huge declared grid stays fast. +const MAX_BLOCKS: u32 = 64; + +pub fn run(data: &[u8]) { + let Ok(mut reader) = GeoTiffReader::open(Cursor::new(data), RasterLimits::new(BLOCK_BYTES)) + else { + return; + }; + let _ = reader.srid(); + let bands = usize::from(reader.band_count()); + let sample_type = reader.sample_type(); + for index in 0..reader.block_count().min(MAX_BLOCKS) { + let Ok(block) = reader.read_block(index) else { + continue; + }; + assert_eq!(block.bands.len(), bands, "band count of block {index}"); + for samples in &block.bands { + assert_eq!(samples.len(), block.window.pixels(), "block {index}"); + assert_eq!(samples.sample_type(), sample_type, "block {index}"); + } + } +} diff --git a/fuzz/src/targets/mod.rs b/fuzz/src/targets/mod.rs index 184666b71..12721a036 100644 --- a/fuzz/src/targets/mod.rs +++ b/fuzz/src/targets/mod.rs @@ -1,5 +1,6 @@ pub mod codec_decoders; pub mod columnar_segment; +pub mod geotiff_decoder; pub mod msgpack_document; pub mod sql_preprocess; pub mod strict_tuple; diff --git a/nodedb-array/Cargo.toml b/nodedb-array/Cargo.toml index 05c22889f..315332b3f 100644 --- a/nodedb-array/Cargo.toml +++ b/nodedb-array/Cargo.toml @@ -15,6 +15,7 @@ categories = ["science", "database-implementations"] [dependencies] nodedb-types = { workspace = true } nodedb-codec = { workspace = true } +nodedb-spatial = { workspace = true } serde = { workspace = true } zerompk = { workspace = true } thiserror = { workspace = true } diff --git a/nodedb-array/src/error.rs b/nodedb-array/src/error.rs index 5d17ce8cc..de157476b 100644 --- a/nodedb-array/src/error.rs +++ b/nodedb-array/src/error.rs @@ -41,6 +41,15 @@ pub enum ArrayError { #[error("tile-extent vector rejected on '{array}': {detail}")] InvalidTileExtents { array: String, detail: String }, + /// A georef names a missing or ill-typed dim, or carries a degenerate + /// transform. + #[error("georef rejected on '{array}': {detail}")] + InvalidGeoref { array: String, detail: String }, + + /// A zone with an SRID cannot be framed over the array's dims. + #[error(transparent)] + ZoneFrame(#[from] crate::query::zone_frame::ZoneFrameError), + #[error("coordinate arity {got} does not match schema arity {expected} on '{array}'")] CoordArityMismatch { array: String, @@ -149,10 +158,12 @@ impl ArrayError { | ArrayError::InvalidDim { array, .. } | ArrayError::InvalidAttr { array, .. } | ArrayError::InvalidTileExtents { array, .. } + | ArrayError::InvalidGeoref { array, .. } | ArrayError::CoordArityMismatch { array, .. } | ArrayError::CoordOutOfDomain { array, .. } | ArrayError::CellTypeMismatch { array, .. } | ArrayError::Narrow { array, .. } => array, + ArrayError::ZoneFrame(e) => e.array(), ArrayError::NumericOverflow { .. } | ArrayError::SegmentCorruption { .. } | ArrayError::UnsupportedFormat { .. } diff --git a/nodedb-array/src/lib.rs b/nodedb-array/src/lib.rs index 08b35c6be..fe86d2825 100644 --- a/nodedb-array/src/lib.rs +++ b/nodedb-array/src/lib.rs @@ -24,7 +24,8 @@ pub use coord::{ }; pub use error::{ArrayError, ArrayResult}; pub use schema::{ - ArraySchema, ArraySchemaBuilder, AttrSpec, CellOrder, DimSpec, DimType, TileOrder, + ArrayGeoref, ArraySchema, ArraySchemaBuilder, AttrSpec, CellOrder, DimSpec, DimType, + GeoTransform, TileOrder, }; pub use segment::{ FOOTER_MAGIC, FORMAT_VERSION, HEADER_MAGIC, HilbertPackedRTree, MbrQueryPredicate, diff --git a/nodedb-array/src/query/mod.rs b/nodedb-array/src/query/mod.rs index 19e09ea18..cc898231a 100644 --- a/nodedb-array/src/query/mod.rs +++ b/nodedb-array/src/query/mod.rs @@ -18,6 +18,7 @@ pub mod rechunk; pub mod region; pub mod retention; pub mod slice; +pub mod zone_frame; pub use agg_number::{AggNumber, AggScalar}; pub use aggregate::{AggregateResult, GroupAggregate, Reducer, aggregate_attr, group_by_dim}; @@ -25,9 +26,14 @@ pub use ceiling::{CeilingParams, CeilingResult, ceiling_resolve_cell}; pub use elementwise::{BinaryOp, elementwise}; pub use project::{Projection, project_sparse}; pub use rechunk::rechunk_sparse; -pub use region::{ArrayRegion, PolygonZone, ZonePolygon, filter_region_owned}; +pub use region::{ + ArrayRegion, PolygonZone, PreparedRegion, PreparedZone, ZonePolygon, filter_region_owned, +}; pub use retention::{RetentionMergeResult, merge_for_retention}; pub use slice::{ DimRange, Slice, filter_sparse, slice_sparse, slice_sparse_owned, tile_overlaps_slice, tile_within_slice, }; +pub use zone_frame::{ + MAX_BISECTION_DEPTH, MAX_FRAME_VERTICES, ZoneFrame, ZoneFrameError, build_zone_frame, +}; diff --git a/nodedb-array/src/query/region.rs b/nodedb-array/src/query/region.rs index b2d4fc39e..b41c2864b 100644 --- a/nodedb-array/src/query/region.rs +++ b/nodedb-array/src/query/region.rs @@ -9,14 +9,20 @@ //! of [`nodedb_types::geometry::point_in_polygon`]: a point on a minimum //! edge is inside, a point on a maximum edge is outside. //! +//! A zone with a [`ZoneFrame`] is drawn in an SRID. Its cells map into +//! that SRID before the ray cast, and its frame's dim box bounds pruning. +//! //! Two entry points: //! * [`ArrayRegion::tile_may_contain`] — MBR-only check. A `false` proves //! the tile holds no live cell in the region, so no payload is decoded. -//! * [`filter_region_owned`] — per-cell filter of one tile. +//! * [`filter_region_owned`] — per-cell filter of one tile, through a +//! [`PreparedRegion`] built once per scan. +use nodedb_spatial::crs::Transformer; use nodedb_types::geometry::point_in_polygon; use super::slice::{Slice, cell_in_slice, filter_sparse, slice_sparse_owned, tile_overlaps_slice}; +use super::zone_frame::ZoneFrame; use crate::error::ArrayResult; use crate::schema::ArraySchema; use crate::tile::mbr::TileMBR; @@ -75,23 +81,27 @@ pub struct PolygonZone { pub x_dim: u32, pub y_dim: u32, pub polygons: Vec, + /// `None`: the polygons are drawn in dim coordinates. `Some`: they are + /// drawn in the frame's zone SRID. + pub frame: Option, } impl PolygonZone { - fn contains(&self, coord: &[CoordValue]) -> bool { - let (Some(x), Some(y)) = ( - coord.get(self.x_dim as usize).and_then(coord_to_f64), - coord.get(self.y_dim as usize).and_then(coord_to_f64), - ) else { - return false; - }; - self.polygons.iter().any(|p| p.contains(x, y)) + /// The zone ready for per-cell tests. Build it once per scan. + pub fn prepare(&self) -> PreparedZone<'_> { + PreparedZone { + zone: self, + frame: self.frame.as_ref().map(|f| (f, f.cell_transformer())), + } } - /// False only when no member polygon's bounding box meets the MBR's - /// `x_dim` × `y_dim` rectangle. A bound that is not numeric proves - /// nothing, so the tile stays in. - fn may_overlap(&self, mbr: &TileMBR) -> bool { + /// False only when the zone's dim extent misses the MBR's `x_dim` × + /// `y_dim` rectangle. A bound that is not numeric proves nothing, so the + /// tile stays in. + /// + /// The dim extent is the frame's dim box, or each member polygon's + /// bounding box when the zone has no frame. + pub fn may_overlap(&self, mbr: &TileMBR) -> bool { let bound = |dims: &[DomainBound], dim: u32| dims.get(dim as usize).and_then(bound_to_f64); let (Some(tx0), Some(ty0), Some(tx1), Some(ty1)) = ( bound(&mbr.dim_mins, self.x_dim), @@ -101,10 +111,42 @@ impl PolygonZone { ) else { return true; }; - self.polygons - .iter() - .filter_map(ZonePolygon::bbox) - .any(|[px0, py0, px1, py1]| px0 <= tx1 && tx0 <= px1 && py0 <= ty1 && ty0 <= py1) + let meets = + |[px0, py0, px1, py1]: [f64; 4]| px0 <= tx1 && tx0 <= px1 && py0 <= ty1 && ty0 <= py1; + match &self.frame { + Some(frame) => meets(frame.dim_bbox), + None => self + .polygons + .iter() + .filter_map(ZonePolygon::bbox) + .any(meets), + } + } +} + +/// A [`PolygonZone`] with its cell-to-zone transformer built. +pub struct PreparedZone<'a> { + zone: &'a PolygonZone, + frame: Option<(&'a ZoneFrame, Transformer)>, +} + +impl PreparedZone<'_> { + /// True if the cell at `coord` lies in the zone. + /// + /// With a frame, the cell centre maps into the zone SRID first. A cell + /// the projection cannot map is outside. + pub fn contains(&self, coord: &[CoordValue]) -> bool { + let (Some(x), Some(y)) = ( + coord.get(self.zone.x_dim as usize), + coord.get(self.zone.y_dim as usize), + ) else { + return false; + }; + let point = match &self.frame { + Some((frame, transformer)) => frame.cell_to_zone(transformer, x, y), + None => coord_to_f64(x).zip(coord_to_f64(y)).map(|(x, y)| [x, y]), + }; + point.is_some_and(|[x, y]| self.zone.polygons.iter().any(|p| p.contains(x, y))) } } @@ -136,9 +178,31 @@ impl ArrayRegion { && self.zone.as_ref().is_none_or(|z| z.may_overlap(mbr)) } + /// The region ready for per-cell tests. Build it once per scan. + pub fn prepare(&self) -> PreparedRegion<'_> { + PreparedRegion { + region: self, + zone: self.zone.as_ref().map(PolygonZone::prepare), + } + } +} + +/// An [`ArrayRegion`] with its zone prepared. +pub struct PreparedRegion<'a> { + region: &'a ArrayRegion, + zone: Option>, +} + +impl PreparedRegion<'_> { + /// The region this was prepared from. + pub fn region(&self) -> &ArrayRegion { + self.region + } + /// True if the cell at `coord` lies in the region. pub fn contains(&self, coord: &[CoordValue]) -> bool { - cell_in_slice(coord, &self.slice) && self.zone.as_ref().is_none_or(|z| z.contains(coord)) + cell_in_slice(coord, &self.region.slice) + && self.zone.as_ref().is_none_or(|z| z.contains(coord)) } } @@ -147,18 +211,19 @@ impl ArrayRegion { pub fn filter_region_owned( schema: &ArraySchema, tile: SparseTile, - region: &ArrayRegion, + region: &PreparedRegion<'_>, ) -> ArrayResult { - if !region.tile_may_contain(&tile.mbr) { + if !region.region.tile_may_contain(&tile.mbr) { return Ok(SparseTile::empty(schema)); } match ®ion.zone { - None => slice_sparse_owned(schema, tile, ®ion.slice), + None => slice_sparse_owned(schema, tile, ®ion.region.slice), Some(_) => filter_sparse(schema, &tile, |coord| region.contains(coord)), } } -fn coord_to_f64(c: &CoordValue) -> Option { +/// A numeric coordinate as `f64`. `None` for a timestamp or string. +pub(crate) fn coord_to_f64(c: &CoordValue) -> Option { match c { CoordValue::Int64(v) => Some(*v as f64), CoordValue::Float64(v) => Some(*v), @@ -184,6 +249,7 @@ mod tests { use crate::tile::sparse_tile::SparseTileBuilder; use crate::types::cell_value::value::CellValue; use crate::types::domain::Domain; + use nodedb_types::geometry::Srid; fn schema() -> ArraySchema { ArraySchemaBuilder::new("g") @@ -227,14 +293,110 @@ mod tests { x_dim: 0, y_dim: 1, polygons, + frame: None, }), ) } + /// One-off membership for tests. A scan prepares the region once. + trait Contains { + fn contains(&self, coord: &[CoordValue]) -> bool; + } + + impl Contains for ArrayRegion { + fn contains(&self, coord: &[CoordValue]) -> bool { + self.prepare().contains(coord) + } + } + fn at(x: f64, y: f64) -> [CoordValue; 2] { [CoordValue::Float64(x), CoordValue::Float64(y)] } + /// A zone drawn in `zone_srid` over a world-mode array in `array_srid`. + fn framed(array_srid: Srid, zone_srid: Srid, polygons: Vec) -> ArrayRegion { + ArrayRegion::new( + Slice::default(), + Some(PolygonZone { + x_dim: 0, + y_dim: 1, + polygons, + frame: Some(ZoneFrame { + transform: None, + array_srid, + zone_srid, + dim_bbox: [-180.0, -90.0, 180.0, 90.0], + }), + }), + ) + } + + #[test] + fn a_framed_zone_keeps_the_edge_and_hole_rules() { + let r = framed( + Srid::Wgs84, + Srid::Wgs84, + vec![ZonePolygon { + exterior: square(0.0, 10.0), + holes: vec![square(4.0, 6.0)], + }], + ); + assert!(r.contains(&at(0.0, 2.0)), "min x edge"); + assert!(r.contains(&at(2.0, 0.0)), "min y edge"); + assert!(!r.contains(&at(10.0, 2.0)), "max x edge"); + assert!(!r.contains(&at(2.0, 10.0)), "max y edge"); + assert!(r.contains(&at(2.0, 2.0))); + assert!(!r.contains(&at(5.0, 5.0)), "inside the hole"); + assert!( + !r.contains(&at(4.0, 5.0)), + "on the hole's min edge, so in the hole" + ); + assert!( + r.contains(&at(6.0, 5.0)), + "on the hole's max edge, so outside it" + ); + } + + #[test] + fn a_cell_the_projection_cannot_map_is_outside() { + let r = framed( + Srid::Wgs84, + Srid::Utm47N, + vec![ZonePolygon { + exterior: square(-1.0e7, 1.0e7), + holes: Vec::new(), + }], + ); + assert!(r.contains(&at(99.5, 6.0)), "inside the UTM 47N domain"); + assert!( + !r.contains(&at(150.0, 6.0)), + "51° from the central meridian" + ); + assert!(!r.contains(&[CoordValue::Float64(f64::NAN), CoordValue::Float64(6.0)])); + } + + #[test] + fn a_framed_zone_prunes_by_its_dim_box() { + let mut r = framed( + Srid::Wgs84, + Srid::Wgs84, + vec![ZonePolygon { + exterior: square(0.0, 50.0), + holes: Vec::new(), + }], + ); + if let Some(zone) = r.zone.as_mut() + && let Some(frame) = zone.frame.as_mut() + { + frame.dim_bbox = [0.0, 0.0, 4.0, 4.0]; + } + assert!(r.tile_may_contain(&tile(&[(1.0, 1.0)]).mbr)); + assert!( + !r.tile_may_contain(&tile(&[(20.0, 20.0)]).mbr), + "the dim box prunes, not the polygon's own box" + ); + } + #[test] fn a_minimum_edge_is_in_and_a_maximum_edge_is_out() { let r = zone(vec![ZonePolygon { @@ -311,11 +473,13 @@ mod tests { exterior: square(0.0, 4.0), holes: Vec::new(), }]); + let r = r.prepare(); let out = filter_region_owned(&s, tile(&[(1.0, 1.0), (0.0, 3.0), (4.0, 1.0)]), &r).unwrap(); assert_eq!(out.nnz(), 2); let pruned = filter_region_owned(&s, tile(&[(6.0, 6.0)]), &r).unwrap(); assert_eq!(pruned.nnz(), 0); - let whole = filter_region_owned(&s, tile(&[(6.0, 6.0)]), &ArrayRegion::default()).unwrap(); + let whole = ArrayRegion::default(); + let whole = filter_region_owned(&s, tile(&[(6.0, 6.0)]), &whole.prepare()).unwrap(); assert_eq!(whole.nnz(), 1); } diff --git a/nodedb-array/src/query/zone_frame.rs b/nodedb-array/src/query/zone_frame.rs new file mode 100644 index 000000000..b94948cf2 --- /dev/null +++ b/nodedb-array/src/query/zone_frame.rs @@ -0,0 +1,634 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The frame of a zone drawn in an SRID: how a cell reaches the zone's SRID, +//! and the dim box that bounds the zone for tile pruning. +//! +//! Membership is exact per cell. The cell centre maps into the array CRS, +//! through the affine in pixel mode. It then maps into the zone SRID, and ray +//! casting runs there. A zone edge is straight in the zone SRID, so its image +//! in dim space is curved. The dim box densifies each exterior edge by +//! recursive bisection before it takes the bounds. + +use nodedb_spatial::crs::Transformer; +use nodedb_types::geometry::Srid; +use serde::{Deserialize, Serialize}; + +use super::region::{ZonePolygon, coord_to_f64}; +use crate::schema::{ArraySchema, DimType, GeoTransform}; +use crate::types::coord::value::CoordValue; + +/// Most positions one dim box may map, summed over every edge. +pub const MAX_FRAME_VERTICES: usize = 1 << 20; + +/// Most bisection levels one edge may take. +pub const MAX_BISECTION_DEPTH: u32 = 24; + +/// Chord tolerance in pixel mode, in pixels. +const PIXEL_TOLERANCE: f64 = 0.25; + +/// World-mode tolerance is the smallest X/Y tile extent over this divisor. +const WORLD_TOLERANCE_DIVISOR: f64 = 64.0; + +/// Offset from a pixel corner to its centre, in pixels. +const PIXEL_CENTRE: f64 = 0.5; + +/// The SRIDs and affine that carry a cell into the zone's SRID. +#[derive( + Debug, + Clone, + PartialEq, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct ZoneFrame { + /// The georef affine. `None` means the dims hold world coordinates. + pub transform: Option, + pub array_srid: Srid, + pub zone_srid: Srid, + /// `[min_x, min_y, max_x, max_y]` in dim coordinates. + /// + /// Every cell the zone holds lies inside it. Pruning reads it. Membership + /// never does. + pub dim_bbox: [f64; 4], +} + +impl ZoneFrame { + /// The cell-to-zone transformer. Build one per scan, never one per cell. + pub fn cell_transformer(&self) -> Transformer { + Transformer::new(self.array_srid, self.zone_srid) + } + + /// Position in the zone SRID of the centre of the cell at dim + /// coordinates `(x, y)`. + /// + /// `None` when a coordinate has the wrong type or a projection cannot + /// map the centre. + pub fn cell_to_zone( + &self, + transformer: &Transformer, + x: &CoordValue, + y: &CoordValue, + ) -> Option<[f64; 2]> { + let world = match &self.transform { + Some(gt) => match (x, y) { + (CoordValue::Int64(i), CoordValue::Int64(j)) => { + let (wx, wy) = gt.cell_centre(*i, *j); + [wx, wy] + } + _ => return None, + }, + None => [coord_to_f64(x)?, coord_to_f64(y)?], + }; + transformer.apply(world).ok() + } +} + +/// Why a zone frame cannot be built. +#[derive(Debug, Clone, PartialEq, thiserror::Error)] +pub enum ZoneFrameError { + /// A zone with an SRID needs the array's GEOREF. SQLSTATE `55000`. + #[error("array '{array}' has no GEOREF; a zone with an SRID needs one")] + NoGeoref { array: String }, + + /// Bounding the zone edges needs more positions than the budget. + /// SQLSTATE `54000`. + #[error( + "zone on array '{array}' needs more than {limit} positions to bound its edges; \ + simplify the zone or split it" + )] + VertexBudget { array: String, limit: usize }, + + /// A zone position cannot map into the array's dims. SQLSTATE `22000`. + #[error( + "zone position ({x}, {y}) in SRID {zone_srid} cannot map into the dims of \ + array '{array}': {detail}" + )] + OutOfDomain { + array: String, + x: f64, + y: f64, + zone_srid: Srid, + detail: String, + }, + + /// The zone has no exterior position. SQLSTATE `22000`. + #[error("zone on array '{array}' has no exterior position")] + EmptyZone { array: String }, +} + +impl ZoneFrameError { + /// The array the frame was built for. + pub fn array(&self) -> &str { + match self { + ZoneFrameError::NoGeoref { array } + | ZoneFrameError::VertexBudget { array, .. } + | ZoneFrameError::OutOfDomain { array, .. } + | ZoneFrameError::EmptyZone { array } => array, + } + } +} + +/// Build the frame of `polygons`, drawn in `zone_srid`, over `schema`. +/// +/// The dim box is padded by twice the chord tolerance. An `INT64` dim rounds +/// it outward to whole coordinates. +pub fn build_zone_frame( + schema: &ArraySchema, + zone_srid: Srid, + polygons: &[ZonePolygon], +) -> Result { + build_with_budget(schema, zone_srid, polygons, MAX_FRAME_VERTICES) +} + +fn build_with_budget( + schema: &ArraySchema, + zone_srid: Srid, + polygons: &[ZonePolygon], + budget: usize, +) -> Result { + let georef = schema + .georef + .as_ref() + .ok_or_else(|| ZoneFrameError::NoGeoref { + array: schema.name.clone(), + })?; + let tolerance = match georef.transform { + Some(_) => PIXEL_TOLERANCE, + None => world_tolerance(schema, georef.x_dim, georef.y_dim), + }; + let mut walk = EdgeWalk { + array: &schema.name, + transformer: Transformer::new(zone_srid, georef.srid), + transform: georef.transform, + zone_srid, + tolerance, + budget, + mapped: 0, + bbox: None, + }; + for polygon in polygons { + walk.ring(&polygon.exterior)?; + } + let [x0, y0, x1, y1] = walk.bbox.ok_or_else(|| ZoneFrameError::EmptyZone { + array: schema.name.clone(), + })?; + let pad = 2.0 * tolerance; + let mut dim_bbox = [x0 - pad, y0 - pad, x1 + pad, y1 + pad]; + if is_int64(schema, georef.x_dim) { + dim_bbox[0] = dim_bbox[0].floor(); + dim_bbox[2] = dim_bbox[2].ceil(); + } + if is_int64(schema, georef.y_dim) { + dim_bbox[1] = dim_bbox[1].floor(); + dim_bbox[3] = dim_bbox[3].ceil(); + } + Ok(ZoneFrame { + transform: georef.transform, + array_srid: georef.srid, + zone_srid, + dim_bbox, + }) +} + +/// The smallest X/Y tile extent over [`WORLD_TOLERANCE_DIVISOR`]. +fn world_tolerance(schema: &ArraySchema, x_dim: u32, y_dim: u32) -> f64 { + let extent = [x_dim, y_dim] + .iter() + .filter_map(|&d| schema.tile_extents.get(d as usize).copied()) + .min() + .unwrap_or(1) + .max(1); + extent as f64 / WORLD_TOLERANCE_DIVISOR +} + +fn is_int64(schema: &ArraySchema, dim: u32) -> bool { + schema + .dims + .get(dim as usize) + .is_some_and(|d| d.dtype == DimType::Int64) +} + +fn midpoint(a: [f64; 2], b: [f64; 2]) -> [f64; 2] { + [(a[0] + b[0]) / 2.0, (a[1] + b[1]) / 2.0] +} + +/// Maps zone positions into dim space and keeps their bounds. +struct EdgeWalk<'a> { + array: &'a str, + transformer: Transformer, + transform: Option, + zone_srid: Srid, + tolerance: f64, + budget: usize, + mapped: usize, + bbox: Option<[f64; 4]>, +} + +impl EdgeWalk<'_> { + /// Bound one ring: every vertex, then every edge, the closing edge too. + fn ring(&mut self, ring: &[[f64; 2]]) -> Result<(), ZoneFrameError> { + let Some((&first, rest)) = ring.split_first() else { + return Ok(()); + }; + let first_dim = self.map(first)?; + let mut prev = (first, first_dim); + for &p in rest { + let p_dim = if p == first { first_dim } else { self.map(p)? }; + self.edge(prev.0, prev.1, p, p_dim, 0)?; + prev = (p, p_dim); + } + if prev.0 != first { + self.edge(prev.0, prev.1, first, first_dim, 0)?; + } + Ok(()) + } + + /// Bisect edge `a → b` until the mapped midpoint lies within the + /// tolerance of the mapped chord's midpoint, or the depth cap is reached. + fn edge( + &mut self, + a: [f64; 2], + a_dim: [f64; 2], + b: [f64; 2], + b_dim: [f64; 2], + depth: u32, + ) -> Result<(), ZoneFrameError> { + if depth >= MAX_BISECTION_DEPTH { + return Ok(()); + } + let mid = midpoint(a, b); + let mid_dim = self.map(mid)?; + let chord = midpoint(a_dim, b_dim); + if (mid_dim[0] - chord[0]).hypot(mid_dim[1] - chord[1]) <= self.tolerance { + return Ok(()); + } + self.edge(a, a_dim, mid, mid_dim, depth + 1)?; + self.edge(mid, mid_dim, b, b_dim, depth + 1) + } + + /// Map one zone position into dim coordinates and extend the bounds. + fn map(&mut self, p: [f64; 2]) -> Result<[f64; 2], ZoneFrameError> { + if self.mapped >= self.budget { + return Err(ZoneFrameError::VertexBudget { + array: self.array.to_string(), + limit: self.budget, + }); + } + self.mapped += 1; + let world = self + .transformer + .apply(p) + .map_err(|e| self.out_of_domain(p, e.to_string()))?; + let dim = match &self.transform { + None => world, + Some(gt) => { + let (u, v) = gt + .world_to_pixel(world[0], world[1]) + .ok_or_else(|| self.out_of_domain(p, "the GEOREF has no inverse".into()))?; + [u - PIXEL_CENTRE, v - PIXEL_CENTRE] + } + }; + if !(dim[0].is_finite() && dim[1].is_finite()) { + return Err(self.out_of_domain(p, "the dim position is not finite".into())); + } + self.bbox = Some(match self.bbox { + None => [dim[0], dim[1], dim[0], dim[1]], + Some(b) => [ + b[0].min(dim[0]), + b[1].min(dim[1]), + b[2].max(dim[0]), + b[3].max(dim[1]), + ], + }); + Ok(dim) + } + + fn out_of_domain(&self, p: [f64; 2], detail: String) -> ZoneFrameError { + ZoneFrameError::OutOfDomain { + array: self.array.to_string(), + x: p[0], + y: p[1], + zone_srid: self.zone_srid, + detail, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::query::region::PolygonZone; + use crate::schema::{ArrayGeoref, ArraySchemaBuilder, AttrSpec, AttrType, DimSpec}; + use crate::types::domain::{Domain, DomainBound}; + + /// Pixel size of the UTM grids, metres. + const PIXEL_M: f64 = 30.0; + + fn utm(lon: f64, lat: f64) -> [f64; 2] { + Transformer::new(Srid::Wgs84, Srid::Utm47N) + .apply([lon, lat]) + .unwrap() + } + + fn pixel_schema(gt: [f64; 6]) -> ArraySchema { + let dim = |name: &str| { + DimSpec::new( + name, + DimType::Int64, + Domain::new(DomainBound::Int64(0), DomainBound::Int64(1 << 20)), + ) + }; + ArraySchemaBuilder::new("scene") + .dim(dim("row")) + .dim(dim("col")) + .attr(AttrSpec::new("b1", AttrType::Float64, true)) + .tile_extents(vec![256, 256]) + .build() + .unwrap() + .with_georef(Some(ArrayGeoref { + srid: Srid::Utm47N, + x_dim: 1, + y_dim: 0, + transform: Some(GeoTransform(gt)), + })) + .unwrap() + } + + fn world_schema(srid: Srid, lo: f64, hi: f64, extent: u64) -> ArraySchema { + let dim = |name: &str| { + DimSpec::new( + name, + DimType::Float64, + Domain::new(DomainBound::Float64(lo), DomainBound::Float64(hi)), + ) + }; + ArraySchemaBuilder::new("world") + .dim(dim("x")) + .dim(dim("y")) + .attr(AttrSpec::new("b1", AttrType::Float64, true)) + .tile_extents(vec![extent, extent]) + .build() + .unwrap() + .with_georef(Some(ArrayGeoref { + srid, + x_dim: 0, + y_dim: 1, + transform: None, + })) + .unwrap() + } + + fn rect(x0: f64, y0: f64, x1: f64, y1: f64) -> ZonePolygon { + ZonePolygon { + exterior: vec![[x0, y0], [x1, y0], [x1, y1], [x0, y1], [x0, y0]], + holes: Vec::new(), + } + } + + /// The 30 m UTM 47N grid around the 6°N parallel between 99°E and 100°E. + /// Returns the schema and its column and row counts. + fn grid_at_six_north() -> (ArraySchema, i64, i64) { + let west = utm(98.995, 6.0)[0].floor(); + let east = utm(100.005, 6.0)[0].ceil(); + let north = utm(100.005, 6.004)[1].ceil(); + let south = utm(99.0, 5.996)[1].floor(); + let cols = ((east - west) / PIXEL_M).ceil() as i64; + let rows = ((north - south) / PIXEL_M).ceil() as i64; + let schema = pixel_schema([west, PIXEL_M, 0.0, north, 0.0, -PIXEL_M]); + (schema, cols, rows) + } + + #[test] + fn a_curved_edge_bounds_every_cell_inside_it() { + let (schema, cols, rows) = grid_at_six_north(); + let polygons = vec![rect(99.0, 6.0, 100.0, 6.5)]; + let frame = build_zone_frame(&schema, Srid::Wgs84, &polygons).unwrap(); + let zone = PolygonZone { + x_dim: 1, + y_dim: 0, + polygons, + frame: Some(frame.clone()), + }; + let prepared = zone.prepare(); + let mut inside = 0usize; + for row in 0..rows { + for col in 0..cols { + if prepared.contains(&[CoordValue::Int64(row), CoordValue::Int64(col)]) { + inside += 1; + let [x0, y0, x1, y1] = frame.dim_bbox; + let (c, r) = (col as f64, row as f64); + assert!( + x0 <= c && c <= x1 && y0 <= r && r <= y1, + "cell ({row}, {col}) is inside the zone but outside {:?}", + frame.dim_bbox + ); + } + } + } + assert!(inside > 0, "the grid holds cells inside the zone"); + } + + /// The 6°N edge from 99°E to 100°E is straight in 4326. Its UTM image + /// bows up to 25 m south of the chord between its projected vertices. + #[test] + fn exact_membership_follows_the_parallel_and_matches_per_cell_truth() { + let (schema, cols, rows) = grid_at_six_north(); + let gt = schema.georef.as_ref().unwrap().transform.unwrap(); + let polygons = vec![rect(99.0, 6.0, 100.0, 6.5)]; + let frame = build_zone_frame(&schema, Srid::Wgs84, &polygons).unwrap(); + let zone = PolygonZone { + x_dim: 1, + y_dim: 0, + polygons: polygons.clone(), + frame: Some(frame), + }; + let prepared = zone.prepare(); + let straight: Vec<[f64; 2]> = polygons[0] + .exterior + .iter() + .map(|p| utm(p[0], p[1])) + .collect(); + let to_lon_lat = Transformer::new(Srid::Utm47N, Srid::Wgs84); + let (mut differs, mut inside) = (0usize, 0usize); + for row in 0..rows { + for col in 0..cols { + let exact = prepared.contains(&[CoordValue::Int64(row), CoordValue::Int64(col)]); + let (x, y) = gt.cell_centre(col, row); + let [lon, lat] = to_lon_lat.apply([x, y]).unwrap(); + let truth = (99.0..100.0).contains(&lon) && (6.0..6.5).contains(&lat); + assert_eq!(exact, truth, "cell ({row}, {col}) at ({lon}, {lat})"); + let vertex_only = nodedb_types::geometry::point_in_polygon(x, y, &straight); + differs += usize::from(exact != vertex_only); + inside += usize::from(exact); + } + } + assert!(inside > 0, "the grid holds cells inside the zone"); + assert!( + differs > 100, + "only {differs} cells differ from the straight UTM edges" + ); + } + + #[test] + fn the_dim_bbox_follows_the_parallel_not_the_chord() { + let (schema, _, _) = grid_at_six_north(); + // The south edge straddles the central meridian at 99°E. In UTM the + // parallel dips south at the meridian, between the two vertices. + let frame = + build_zone_frame(&schema, Srid::Wgs84, &[rect(98.5, 6.0, 99.5, 6.001)]).unwrap(); + let gt = schema.georef.as_ref().unwrap().transform.unwrap(); + let row_of = |lon: f64| { + let p = utm(lon, 6.0); + gt.world_to_pixel(p[0], p[1]).unwrap().1 - 0.5 + }; + let vertex_row = row_of(98.5).max(row_of(99.5)); + let bulge_row = row_of(99.0); + assert!( + bulge_row > vertex_row + 2.0 * PIXEL_TOLERANCE, + "the bulge passes the padded vertex box: {bulge_row} vs {vertex_row}" + ); + assert!(frame.dim_bbox[3] >= bulge_row, "{:?}", frame.dim_bbox); + assert!( + frame.dim_bbox.iter().all(|b| b.fract() == 0.0), + "INT64 dims round" + ); + } + + #[test] + fn a_world_frame_pads_by_the_tile_tolerance() { + let schema = world_schema(Srid::Wgs84, -180.0, 180.0, 64); + let frame = + build_zone_frame(&schema, Srid::Wgs84, &[rect(100.0, 2.0, 101.0, 3.0)]).unwrap(); + // Same SRID: the edges map straight, and the pad is 2 · 64 / 64. + assert_eq!(frame.dim_bbox, [98.0, 0.0, 103.0, 5.0]); + assert_eq!(frame.transform, None); + } + + #[test] + fn a_zone_without_georef_is_refused() { + let mut schema = world_schema(Srid::Wgs84, 0.0, 10.0, 1); + schema.georef = None; + let err = build_zone_frame(&schema, Srid::Wgs84, &[rect(1.0, 1.0, 2.0, 2.0)]) + .expect_err("refused"); + assert!(matches!(err, ZoneFrameError::NoGeoref { .. }), "{err:?}"); + } + + #[test] + fn a_vertex_outside_the_projection_is_refused() { + let (schema, _, _) = grid_at_six_north(); + let err = build_zone_frame(&schema, Srid::Wgs84, &[rect(150.0, 6.0, 151.0, 7.0)]) + .expect_err("refused"); + assert!(matches!(err, ZoneFrameError::OutOfDomain { .. }), "{err:?}"); + } + + #[test] + fn the_vertex_budget_is_enforced() { + let (schema, _, _) = grid_at_six_north(); + let polygons = [rect(99.0, 6.0, 100.0, 6.5)]; + assert!(build_with_budget(&schema, Srid::Wgs84, &polygons, 1 << 16).is_ok()); + let err = build_with_budget(&schema, Srid::Wgs84, &polygons, 8).expect_err("over budget"); + assert_eq!( + err, + ZoneFrameError::VertexBudget { + array: "scene".into(), + limit: 8 + } + ); + } + + #[test] + fn a_sub_micron_pixel_exhausts_the_default_budget() { + let origin = utm(99.0, 6.5); + let schema = pixel_schema([origin[0], 1e-9, 0.0, origin[1], 0.0, -1e-9]); + let ring = ZonePolygon { + exterior: vec![ + [99.0, 6.0], + [100.0, 6.0], + [101.0, 6.0], + [102.0, 6.0], + [102.0, 6.5], + [101.0, 6.5], + [100.0, 6.5], + [99.0, 6.5], + [99.0, 6.0], + ], + holes: Vec::new(), + }; + let err = build_zone_frame(&schema, Srid::Wgs84, &[ring]).expect_err("over budget"); + assert!( + matches!(err, ZoneFrameError::VertexBudget { .. }), + "{err:?}" + ); + } + + /// Deterministic pseudo-random numbers in `[0, 1)`. + struct Lcg(u64); + + impl Lcg { + fn next(&mut self) -> f64 { + self.0 = self + .0 + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + (self.0 >> 11) as f64 / (1u64 << 53) as f64 + } + } + + #[test] + fn every_inside_cell_lies_in_the_dim_bbox_for_random_zones() { + let (schema, cols, rows) = grid_at_six_north(); + let gt = schema.georef.as_ref().unwrap().transform.unwrap(); + let to_lon_lat = Transformer::new(Srid::Utm47N, Srid::Wgs84); + let mut rng = Lcg(0x5eed); + for case in 0..24 { + // A random quadrilateral whose corners fall inside the grid. + let corner = |rng: &mut Lcg| { + let (x, y) = gt.pixel_to_world(rng.next() * cols as f64, rng.next() * rows as f64); + to_lon_lat.apply([x, y]).unwrap() + }; + let a = corner(&mut rng); + let b = corner(&mut rng); + let c = corner(&mut rng); + let d = corner(&mut rng); + let polygons = vec![ZonePolygon { + exterior: vec![a, b, c, d, a], + holes: Vec::new(), + }]; + let frame = build_zone_frame(&schema, Srid::Wgs84, &polygons).unwrap(); + let [x0, y0, x1, y1] = frame.dim_bbox; + let zone = PolygonZone { + x_dim: 1, + y_dim: 0, + polygons, + frame: Some(frame), + }; + let prepared = zone.prepare(); + for row in (0..rows).step_by(3) { + for col in (0..cols).step_by(7) { + if prepared.contains(&[CoordValue::Int64(row), CoordValue::Int64(col)]) { + let (c, r) = (col as f64, row as f64); + assert!( + x0 <= c && c <= x1 && y0 <= r && r <= y1, + "case {case}: cell ({row}, {col}) outside [{x0}, {y0}, {x1}, {y1}]" + ); + } + } + } + } + } + + #[test] + fn a_frame_round_trips_through_msgpack() { + let frame = ZoneFrame { + transform: Some(GeoTransform([1.0, 2.0, 0.0, 3.0, 0.0, -2.0])), + array_srid: Srid::Utm47N, + zone_srid: Srid::Wgs84, + dim_bbox: [0.0, 1.0, 2.0, 3.0], + }; + let bytes = zerompk::to_msgpack_vec(&frame).unwrap(); + let back: ZoneFrame = zerompk::from_msgpack(&bytes).unwrap(); + assert_eq!(back, frame); + } +} diff --git a/nodedb-array/src/schema/array_schema.rs b/nodedb-array/src/schema/array_schema.rs index 6c06f3d07..abd394cf1 100644 --- a/nodedb-array/src/schema/array_schema.rs +++ b/nodedb-array/src/schema/array_schema.rs @@ -14,6 +14,9 @@ use serde::{Deserialize, Serialize}; use super::attr_spec::AttrSpec; use super::cell_order::{CellOrder, TileOrder}; use super::dim_spec::DimSpec; +use super::georef::ArrayGeoref; +use super::validation; +use crate::error::ArrayResult; /// Full array schema. #[derive( @@ -35,9 +38,19 @@ pub struct ArraySchema { pub tile_extents: Vec, pub cell_order: CellOrder, pub tile_order: TileOrder, + /// Georeference of the array. Not part of [`Self::content_msgpack`], so + /// it never changes `schema_hash`. + pub georef: Option, } impl ArraySchema { + /// This schema with `georef` in place of its current one, validated + /// against the dims. + pub fn with_georef(self, georef: Option) -> ArrayResult { + validation::georef::check(&self.name, &self.dims, georef.as_ref())?; + Ok(Self { georef, ..self }) + } + pub fn arity(&self) -> usize { self.dims.len() } @@ -103,6 +116,7 @@ mod tests { tile_extents: vec![4], cell_order: CellOrder::Hilbert, tile_order: TileOrder::Hilbert, + georef: None, }; let a = mk("alpha"); let b = mk("beta"); @@ -130,10 +144,68 @@ mod tests { tile_extents: vec![1, 1_000_000], cell_order: CellOrder::Hilbert, tile_order: TileOrder::Hilbert, + georef: None, }; assert_eq!(s.arity(), 2); assert!(s.dim("chrom").is_some()); assert!(s.attr("variant").is_some()); assert!(s.dim("missing").is_none()); } + + fn raster() -> ArraySchema { + let dim = |name: &str| { + DimSpec::new( + name, + DimType::Int64, + Domain::new(DomainBound::Int64(0), DomainBound::Int64(1023)), + ) + }; + ArraySchema { + name: "scene".into(), + dims: vec![dim("row"), dim("col")], + attrs: vec![AttrSpec::new("b1", AttrType::UInt16, true)], + tile_extents: vec![256, 256], + cell_order: CellOrder::RowMajor, + tile_order: TileOrder::RowMajor, + georef: None, + } + } + + fn pixel_georef() -> ArrayGeoref { + ArrayGeoref { + srid: nodedb_types::geometry::Srid::Utm47N, + x_dim: 1, + y_dim: 0, + transform: Some(super::super::georef::GeoTransform([ + 600_000.0, 10.0, 0.0, 700_000.0, 0.0, -10.0, + ])), + } + } + + #[test] + fn content_msgpack_ignores_georef() { + let plain = raster(); + let geo = raster().with_georef(Some(pixel_georef())).unwrap(); + assert_ne!(plain, geo); + assert_eq!(plain.content_msgpack(), geo.content_msgpack()); + } + + #[test] + fn schema_with_georef_msgpack_round_trip() { + let geo = raster().with_georef(Some(pixel_georef())).unwrap(); + let bytes = zerompk::to_msgpack_vec(&geo).unwrap(); + let back: ArraySchema = zerompk::from_msgpack(&bytes).unwrap(); + assert_eq!(back, geo); + assert_eq!(back.georef, Some(pixel_georef())); + } + + #[test] + fn with_georef_validates_and_clears() { + let mut bad = pixel_georef(); + bad.y_dim = 1; + assert!(raster().with_georef(Some(bad)).is_err()); + let geo = raster().with_georef(Some(pixel_georef())).unwrap(); + let cleared = geo.with_georef(None).unwrap(); + assert_eq!(cleared, raster()); + } } diff --git a/nodedb-array/src/schema/builder.rs b/nodedb-array/src/schema/builder.rs index 040a1c95f..2d2c84792 100644 --- a/nodedb-array/src/schema/builder.rs +++ b/nodedb-array/src/schema/builder.rs @@ -11,6 +11,7 @@ use super::array_schema::ArraySchema; use super::attr_spec::AttrSpec; use super::cell_order::{CellOrder, TileOrder}; use super::dim_spec::DimSpec; +use super::georef::ArrayGeoref; use super::validation; use crate::error::ArrayResult; @@ -23,6 +24,7 @@ pub struct ArraySchemaBuilder { tile_extents: Vec, cell_order: CellOrder, tile_order: TileOrder, + georef: Option, } impl ArraySchemaBuilder { @@ -34,9 +36,15 @@ impl ArraySchemaBuilder { tile_extents: Vec::new(), cell_order: CellOrder::default(), tile_order: TileOrder::default(), + georef: None, } } + pub fn georef(mut self, georef: ArrayGeoref) -> Self { + self.georef = Some(georef); + self + } + pub fn dim(mut self, dim: DimSpec) -> Self { self.dims.push(dim); self @@ -66,6 +74,7 @@ impl ArraySchemaBuilder { validation::dims::check(&self.name, &self.dims)?; validation::attrs::check(&self.name, &self.attrs)?; validation::tiles::check(&self.name, &self.dims, &self.tile_extents)?; + validation::georef::check(&self.name, &self.dims, self.georef.as_ref())?; Ok(ArraySchema { name: self.name, dims: self.dims, @@ -73,6 +82,7 @@ impl ArraySchemaBuilder { tile_extents: self.tile_extents, cell_order: self.cell_order, tile_order: self.tile_order, + georef: self.georef, }) } } @@ -123,6 +133,113 @@ mod tests { assert!(r.is_err()); } + fn float_dim(name: &str) -> DimSpec { + DimSpec::new( + name, + DimType::Float64, + Domain::new(DomainBound::Float64(-180.0), DomainBound::Float64(180.0)), + ) + } + + fn geo_builder(x: DimSpec, y: DimSpec) -> ArraySchemaBuilder { + ArraySchemaBuilder::new("scene") + .dim(y) + .dim(x) + .attr(AttrSpec::new("b1", AttrType::UInt16, true)) + .tile_extents(vec![256, 256]) + } + + fn georef(transform: Option<[f64; 6]>) -> ArrayGeoref { + ArrayGeoref { + srid: nodedb_types::geometry::Srid::Wgs84, + x_dim: 1, + y_dim: 0, + transform: transform.map(super::super::georef::GeoTransform), + } + } + + const GT_4326: [f64; 6] = [ + 100.5, + 8.983152841195214e-05, + 0.0, + 6.5, + 0.0, + -8.983152841195214e-05, + ]; + + #[test] + fn build_accepts_pixel_georef() { + let s = geo_builder(int64_dim("col", 1023), int64_dim("row", 1023)) + .georef(georef(Some(GT_4326))) + .build() + .unwrap(); + assert_eq!(s.georef, Some(georef(Some(GT_4326)))); + } + + #[test] + fn build_accepts_world_georef() { + let s = geo_builder(float_dim("lon"), float_dim("lat")) + .georef(georef(None)) + .build() + .unwrap(); + assert!(s.georef.is_some()); + } + + #[test] + fn build_refuses_each_georef_rule() { + // Pixel mode on FLOAT64 dims. + assert!( + geo_builder(float_dim("lon"), float_dim("lat")) + .georef(georef(Some(GT_4326))) + .build() + .is_err() + ); + // World mode on a STRING dim. + let s_dim = DimSpec::new( + "s", + DimType::String, + Domain::new( + DomainBound::String(String::new()), + DomainBound::String("z".into()), + ), + ); + assert!( + geo_builder(s_dim, float_dim("lat")) + .georef(georef(None)) + .build() + .is_err() + ); + // Out-of-range dim index. + let mut missing = georef(None); + missing.x_dim = 2; + assert!( + geo_builder(float_dim("lon"), float_dim("lat")) + .georef(missing) + .build() + .is_err() + ); + // Same dim for both axes. + let mut same = georef(None); + same.y_dim = 1; + assert!( + geo_builder(float_dim("lon"), float_dim("lat")) + .georef(same) + .build() + .is_err() + ); + // Non-finite coefficient and singular transform. + let mut nan = GT_4326; + nan[0] = f64::NAN; + for gt in [nan, [0.0, 1.0, 1.0, 0.0, 1.0, 1.0]] { + assert!( + geo_builder(int64_dim("col", 9), int64_dim("row", 9)) + .georef(georef(Some(gt))) + .build() + .is_err() + ); + } + } + #[test] fn build_rejects_extent_arity_mismatch() { let r = ArraySchemaBuilder::new("g") diff --git a/nodedb-array/src/schema/georef.rs b/nodedb-array/src/schema/georef.rs new file mode 100644 index 000000000..a890c135f --- /dev/null +++ b/nodedb-array/src/schema/georef.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Georeference of an array: which dims carry the x and y axes of a +//! coordinate reference system, and how cell coordinates map to it. +//! +//! The georef stays out of [`super::ArraySchema::content_msgpack`]. Adding, +//! changing, or removing it leaves `schema_hash` and every segment as they are. + +use nodedb_types::geometry::Srid; +use serde::{Deserialize, Serialize}; + +/// Affine pixel-to-world transform in GDAL coefficient order. +/// +/// For a continuous pixel position `(u, v)`, `u` along the column axis and +/// `v` along the row axis: +/// +/// - `X = GT0 + u·GT1 + v·GT2` +/// - `Y = GT3 + u·GT4 + v·GT5` +/// +/// Pixel `(i, j)` covers `[i, i+1) × [j, j+1)`. Its centre is `(i+0.5, j+0.5)`. +#[derive( + Debug, Clone, Copy, Serialize, Deserialize, zerompk::ToMessagePack, zerompk::FromMessagePack, +)] +pub struct GeoTransform(pub [f64; 6]); + +impl PartialEq for GeoTransform { + /// Bitwise equality. Validation admits finite coefficients only, so + /// this is equality of the stored values. + fn eq(&self, other: &Self) -> bool { + self.0 + .iter() + .zip(other.0.iter()) + .all(|(a, b)| a.to_bits() == b.to_bits()) + } +} + +impl Eq for GeoTransform {} + +impl GeoTransform { + /// `GT1·GT5 − GT2·GT4`. Zero means the transform has no inverse. + pub fn determinant(&self) -> f64 { + let [_, gt1, gt2, _, gt4, gt5] = self.0; + gt1 * gt5 - gt2 * gt4 + } + + /// World position of the continuous pixel position `(u, v)`. + pub fn pixel_to_world(&self, u: f64, v: f64) -> (f64, f64) { + let [gt0, gt1, gt2, gt3, gt4, gt5] = self.0; + (gt0 + u * gt1 + v * gt2, gt3 + u * gt4 + v * gt5) + } + + /// World position of the centre of pixel `(i, j)`. + pub fn cell_centre(&self, i: i64, j: i64) -> (f64, f64) { + self.pixel_to_world(i as f64 + 0.5, j as f64 + 0.5) + } + + /// Continuous pixel position of the world position `(x, y)`. + /// + /// Returns `None` when the determinant is zero or not finite. + pub fn world_to_pixel(&self, x: f64, y: f64) -> Option<(f64, f64)> { + let det = self.determinant(); + if det == 0.0 || !det.is_finite() { + return None; + } + let [gt0, gt1, gt2, gt3, gt4, gt5] = self.0; + let dx = x - gt0; + let dy = y - gt3; + Some(((gt5 * dx - gt2 * dy) / det, (gt1 * dy - gt4 * dx) / det)) + } + + /// Index of the pixel that holds the world position `(x, y)`. + /// + /// Returns `None` when the transform has no inverse or the index falls + /// outside `i64`. + pub fn world_to_cell(&self, x: f64, y: f64) -> Option<(i64, i64)> { + let (u, v) = self.world_to_pixel(x, y)?; + Some((floor_to_i64(u)?, floor_to_i64(v)?)) + } +} + +/// `floor(value)` as `i64`. `None` for a non-finite or out-of-range value. +fn floor_to_i64(value: f64) -> Option { + let floored = value.floor(); + // `i64::MAX as f64` rounds up to 2^63, which is outside `i64`. + if floored.is_finite() && floored >= i64::MIN as f64 && floored < i64::MAX as f64 { + Some(floored as i64) + } else { + None + } +} + +/// Georeference of an array. +/// +/// `x_dim` and `y_dim` index into [`super::ArraySchema::dims`]. +/// +/// - `transform: None`: the dims hold pixel-centre world coordinates in +/// `srid`. Each dim is `INT64` or `FLOAT64`. +/// - `transform: Some`: the dims hold `INT64` pixel indices, `x_dim` the +/// column and `y_dim` the row. The transform maps them to world coordinates. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + zerompk::ToMessagePack, + zerompk::FromMessagePack, +)] +pub struct ArrayGeoref { + pub srid: Srid, + pub x_dim: u32, + pub y_dim: u32, + pub transform: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + const EPS: f64 = 1e-9; + + fn assert_close(a: (f64, f64), b: (f64, f64), eps: f64) { + assert!( + (a.0 - b.0).abs() <= eps && (a.1 - b.1).abs() <= eps, + "{a:?} != {b:?}" + ); + } + + #[test] + fn north_up_centre_is_index_plus_half() { + let gt = GeoTransform([100_000.0, 10.0, 0.0, 200_000.0, 0.0, -10.0]); + assert_close(gt.cell_centre(0, 0), (100_005.0, 199_995.0), EPS); + assert_close(gt.cell_centre(3, 2), (100_035.0, 199_975.0), EPS); + } + + #[test] + fn centre_inverse_round_trip_north_up() { + let gt = GeoTransform([100_000.0, 10.0, 0.0, 200_000.0, 0.0, -10.0]); + for (i, j) in [(0, 0), (7, 3), (255, 191), (-4, 9)] { + let (x, y) = gt.cell_centre(i, j); + let (u, v) = gt.world_to_pixel(x, y).unwrap(); + assert_close((u, v), (i as f64 + 0.5, j as f64 + 0.5), EPS); + assert_eq!(gt.world_to_cell(x, y), Some((i, j))); + } + } + + #[test] + fn centre_inverse_round_trip_rotated() { + let gt = GeoTransform([500_000.0, 8.660254, 5.0, 4_000_000.0, 5.0, -8.660254]); + assert!(gt.determinant().abs() > 0.0); + for (i, j) in [(0, 0), (12, 40), (1023, 511)] { + let (x, y) = gt.cell_centre(i, j); + let (u, v) = gt.world_to_pixel(x, y).unwrap(); + assert_close((u, v), (i as f64 + 0.5, j as f64 + 0.5), 1e-6); + assert_eq!(gt.world_to_cell(x, y), Some((i, j))); + } + } + + #[test] + fn centre_inverse_round_trip_geographic_exponent() { + let res = 8.983152841195214e-05; + let gt = GeoTransform([100.5, res, 0.0, 6.5, 0.0, -res]); + let (x, y) = gt.cell_centre(0, 0); + assert_close((x, y), (100.5 + res / 2.0, 6.5 - res / 2.0), 1e-15); + for (i, j) in [(0, 0), (100, 200), (11_131, 5_000)] { + let (x, y) = gt.cell_centre(i, j); + let (u, v) = gt.world_to_pixel(x, y).unwrap(); + assert_close((u, v), (i as f64 + 0.5, j as f64 + 0.5), 1e-6); + assert_eq!(gt.world_to_cell(x, y), Some((i, j))); + } + } + + #[test] + fn singular_transform_has_no_inverse() { + let gt = GeoTransform([0.0, 1.0, 2.0, 0.0, 2.0, 4.0]); + assert_eq!(gt.determinant(), 0.0); + assert!(gt.world_to_pixel(1.0, 1.0).is_none()); + assert!(gt.world_to_cell(1.0, 1.0).is_none()); + } + + #[test] + fn out_of_range_index_is_none() { + let gt = GeoTransform([0.0, 1e-10, 0.0, 0.0, 0.0, -1e-10]); + assert!(gt.world_to_pixel(1e10, -1.0).is_some()); + assert!(gt.world_to_cell(1e10, -1.0).is_none()); + } + + #[test] + fn georef_msgpack_round_trip() { + let g = ArrayGeoref { + srid: Srid::Utm47N, + x_dim: 1, + y_dim: 0, + transform: Some(GeoTransform([600_000.0, 10.0, 0.0, 700_000.0, 0.0, -10.0])), + }; + let bytes = zerompk::to_msgpack_vec(&g).unwrap(); + let back: ArrayGeoref = zerompk::from_msgpack(&bytes).unwrap(); + assert_eq!(back, g); + + let world = ArrayGeoref { + srid: Srid::Wgs84, + x_dim: 0, + y_dim: 1, + transform: None, + }; + let bytes = zerompk::to_msgpack_vec(&world).unwrap(); + let back: ArrayGeoref = zerompk::from_msgpack(&bytes).unwrap(); + assert_eq!(back, world); + } +} diff --git a/nodedb-array/src/schema/mod.rs b/nodedb-array/src/schema/mod.rs index 5772614bb..c2096d48c 100644 --- a/nodedb-array/src/schema/mod.rs +++ b/nodedb-array/src/schema/mod.rs @@ -6,6 +6,7 @@ pub mod builder; pub mod cell_check; pub mod cell_order; pub mod dim_spec; +pub mod georef; pub mod validation; pub use array_schema::ArraySchema; @@ -13,3 +14,4 @@ pub use attr_spec::{AttrSpec, AttrType}; pub use builder::ArraySchemaBuilder; pub use cell_order::{CellOrder, TileOrder}; pub use dim_spec::{DimSpec, DimType}; +pub use georef::{ArrayGeoref, GeoTransform}; diff --git a/nodedb-array/src/schema/validation/georef.rs b/nodedb-array/src/schema/validation/georef.rs new file mode 100644 index 000000000..401446645 --- /dev/null +++ b/nodedb-array/src/schema/validation/georef.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Georef validation. +//! +//! Rules: +//! - `x_dim` and `y_dim` name existing dims and differ. +//! - World mode (no transform): each dim is `INT64` or `FLOAT64`. +//! - Pixel mode (transform): each dim is `INT64`, all six coefficients are +//! finite, and the determinant is finite and non-zero. + +use crate::error::{ArrayError, ArrayResult}; +use crate::schema::dim_spec::{DimSpec, DimType}; +use crate::schema::georef::ArrayGeoref; + +pub fn check(array: &str, dims: &[DimSpec], georef: Option<&ArrayGeoref>) -> ArrayResult<()> { + let Some(georef) = georef else { + return Ok(()); + }; + let refuse = |detail: String| ArrayError::InvalidGeoref { + array: array.to_string(), + detail, + }; + let dim_at = |axis: &str, index: u32| { + usize::try_from(index) + .ok() + .and_then(|i| dims.get(i)) + .ok_or_else(|| { + refuse(format!( + "{axis} dim index {index} is out of range for {} dims", + dims.len() + )) + }) + }; + let x = dim_at("X", georef.x_dim)?; + let y = dim_at("Y", georef.y_dim)?; + if georef.x_dim == georef.y_dim { + return Err(refuse(format!("X and Y name the same dim '{}'", x.name))); + } + match &georef.transform { + None => { + for (axis, dim) in [("X", x), ("Y", y)] { + if !matches!(dim.dtype, DimType::Int64 | DimType::Float64) { + return Err(refuse(format!( + "{axis} dim '{}' is {:?}; world coordinates need INT64 or FLOAT64", + dim.name, dim.dtype + ))); + } + } + } + Some(transform) => { + for (axis, dim) in [("X", x), ("Y", y)] { + if dim.dtype != DimType::Int64 { + return Err(refuse(format!( + "{axis} dim '{}' is {:?}; pixel indices with TRANSFORM need INT64", + dim.name, dim.dtype + ))); + } + } + if let Some(i) = transform.0.iter().position(|c| !c.is_finite()) { + return Err(refuse(format!( + "TRANSFORM coefficient {i} is {}; every coefficient must be finite", + transform.0[i] + ))); + } + let det = transform.determinant(); + if det == 0.0 || !det.is_finite() { + return Err(refuse(format!( + "TRANSFORM determinant GT1*GT5 - GT2*GT4 is {det}; it must be finite and non-zero" + ))); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::schema::georef::GeoTransform; + use crate::types::domain::{Domain, DomainBound}; + use nodedb_types::geometry::Srid; + + fn int_dim(name: &str) -> DimSpec { + DimSpec::new( + name, + DimType::Int64, + Domain::new(DomainBound::Int64(0), DomainBound::Int64(1000)), + ) + } + + fn float_dim(name: &str) -> DimSpec { + DimSpec::new( + name, + DimType::Float64, + Domain::new(DomainBound::Float64(-180.0), DomainBound::Float64(180.0)), + ) + } + + fn string_dim(name: &str) -> DimSpec { + DimSpec::new( + name, + DimType::String, + Domain::new( + DomainBound::String(String::new()), + DomainBound::String("z".into()), + ), + ) + } + + fn georef(x: u32, y: u32, transform: Option<[f64; 6]>) -> ArrayGeoref { + ArrayGeoref { + srid: Srid::Utm47N, + x_dim: x, + y_dim: y, + transform: transform.map(GeoTransform), + } + } + + const NORTH_UP: [f64; 6] = [600_000.0, 10.0, 0.0, 700_000.0, 0.0, -10.0]; + + #[test] + fn none_is_accepted() { + assert!(check("a", &[int_dim("x")], None).is_ok()); + } + + #[test] + fn accepts_world_mode_int_and_float() { + let dims = [float_dim("lon"), int_dim("northing")]; + assert!(check("a", &dims, Some(&georef(0, 1, None))).is_ok()); + } + + #[test] + fn accepts_pixel_mode() { + let dims = [int_dim("row"), int_dim("col")]; + assert!(check("a", &dims, Some(&georef(1, 0, Some(NORTH_UP)))).is_ok()); + } + + #[test] + fn refuses_out_of_range_dim() { + let dims = [int_dim("x"), int_dim("y")]; + assert!(check("a", &dims, Some(&georef(0, 2, None))).is_err()); + assert!(check("a", &dims, Some(&georef(5, 1, None))).is_err()); + } + + #[test] + fn refuses_same_dim() { + let dims = [int_dim("x"), int_dim("y")]; + assert!(check("a", &dims, Some(&georef(1, 1, None))).is_err()); + } + + #[test] + fn refuses_world_mode_string_dim() { + let dims = [string_dim("x"), int_dim("y")]; + assert!(check("a", &dims, Some(&georef(0, 1, None))).is_err()); + } + + #[test] + fn refuses_pixel_mode_float_dim() { + let dims = [float_dim("x"), int_dim("y")]; + assert!(check("a", &dims, Some(&georef(0, 1, Some(NORTH_UP)))).is_err()); + } + + #[test] + fn refuses_non_finite_coefficient() { + let dims = [int_dim("x"), int_dim("y")]; + for bad in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + let mut gt = NORTH_UP; + gt[3] = bad; + assert!(check("a", &dims, Some(&georef(0, 1, Some(gt)))).is_err()); + } + } + + #[test] + fn refuses_singular_transform() { + let dims = [int_dim("x"), int_dim("y")]; + let gt = [0.0, 1.0, 2.0, 0.0, 2.0, 4.0]; + assert!(check("a", &dims, Some(&georef(0, 1, Some(gt)))).is_err()); + } + + #[test] + fn refuses_overflowing_determinant() { + let dims = [int_dim("x"), int_dim("y")]; + let gt = [0.0, 1e300, 0.0, 0.0, 0.0, -1e300]; + assert!(check("a", &dims, Some(&georef(0, 1, Some(gt)))).is_err()); + } +} diff --git a/nodedb-array/src/schema/validation/mod.rs b/nodedb-array/src/schema/validation/mod.rs index 56f848003..4e3126991 100644 --- a/nodedb-array/src/schema/validation/mod.rs +++ b/nodedb-array/src/schema/validation/mod.rs @@ -2,4 +2,5 @@ pub mod attrs; pub mod dims; +pub mod georef; pub mod tiles; diff --git a/nodedb-array/src/sync/schema_crdt.rs b/nodedb-array/src/sync/schema_crdt.rs index 5412c5041..c780d120c 100644 --- a/nodedb-array/src/sync/schema_crdt.rs +++ b/nodedb-array/src/sync/schema_crdt.rs @@ -364,6 +364,7 @@ mod tests { tile_extents: vec![10], cell_order: CellOrder::RowMajor, tile_order: TileOrder::RowMajor, + georef: None, } } diff --git a/nodedb-cluster-tests/tests/common_suite/cases/cluster_array_slice_zone.rs b/nodedb-cluster-tests/tests/common_suite/cases/cluster_array_slice_zone.rs new file mode 100644 index 000000000..45223c396 --- /dev/null +++ b/nodedb-cluster-tests/tests/common_suite/cases/cluster_array_slice_zone.rs @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: BUSL-1.1 +//! 3-node cluster test for `ARRAY_SLICE` with a WGS 84 zone over a +//! georeferenced array. +//! +//! The test matches `binary(/cluster/)` in nextest.toml, so it runs in the +//! serial `cluster` test group. The coordinator forwards the zone to every +//! shard it fans out to. Every node must answer the same row set, and that +//! set must equal the cells the engine's own zone test admits. + +use std::collections::BTreeSet; + +use nodedb_array::query::region::{PolygonZone, ZonePolygon}; +use nodedb_array::query::zone_frame::build_zone_frame; +use nodedb_array::schema::{ + ArrayGeoref, ArraySchemaBuilder, AttrSpec, AttrType, DimSpec, DimType, GeoTransform, +}; +use nodedb_array::types::coord::value::CoordValue; +use nodedb_array::types::domain::{Domain, DomainBound}; +use nodedb_types::geometry::Srid; + +use crate::common; + +use common::cluster_harness::TestCluster; + +const SIZE: i64 = 24; +const GT: [f64; 6] = [550_000.0, 30.0, 0.0, 664_000.0, 0.0, -30.0]; + +/// A WGS 84 quadrilateral over the middle of the grid. +const RING: [[f64; 2]; 5] = [ + [99.4535, 6.0060], + [99.4580, 6.0056], + [99.4575, 6.0022], + [99.4540, 6.0026], + [99.4535, 6.0060], +]; + +fn ewkt() -> String { + let points: Vec = RING + .iter() + .map(|p| format!("{:?} {:?}", p[0], p[1])) + .collect(); + format!("'SRID=4326;POLYGON(({}))'", points.join(", ")) +} + +/// The cells the engine's zone test admits, from the same schema and ring. +fn expected_cells() -> BTreeSet<(i64, i64)> { + let dim = |name: &str| { + DimSpec::new( + name, + DimType::Int64, + Domain::new(DomainBound::Int64(0), DomainBound::Int64(SIZE - 1)), + ) + }; + let schema = ArraySchemaBuilder::new("czone") + .dim(dim("row")) + .dim(dim("col")) + .attr(AttrSpec::new("b1", AttrType::Float64, true)) + .tile_extents(vec![4, 4]) + .build() + .expect("schema") + .with_georef(Some(ArrayGeoref { + srid: Srid::Utm47N, + x_dim: 1, + y_dim: 0, + transform: Some(GeoTransform(GT)), + })) + .expect("georef"); + let polygons = vec![ZonePolygon { + exterior: RING.to_vec(), + holes: Vec::new(), + }]; + let frame = build_zone_frame(&schema, Srid::Wgs84, &polygons).expect("frame"); + let zone = PolygonZone { + x_dim: 1, + y_dim: 0, + polygons, + frame: Some(frame), + }; + let prepared = zone.prepare(); + let mut out = BTreeSet::new(); + for row in 0..SIZE { + for col in 0..SIZE { + if prepared.contains(&[CoordValue::Int64(row), CoordValue::Int64(col)]) { + out.insert((row, col)); + } + } + } + out +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn cluster_array_slice_zone_answers_one_row_set_on_every_node() { + let cluster = TestCluster::spawn_three() + .await + .expect("3-node cluster spawn"); + let ddl_idx = cluster + .exec_ddl_on_any_leader(&format!( + "CREATE ARRAY czone DIMS (row INT64 [0..{last}], col INT64 [0..{last}]) \ + ATTRS (b1 FLOAT64) TILE_EXTENTS (4, 4) \ + GEOREF (SRID 32647, X col, Y row, TRANSFORM ({}, {}, {}, {}, {}, {}))", + GT[0], + GT[1], + GT[2], + GT[3], + GT[4], + GT[5], + last = SIZE - 1 + )) + .await + .expect("CREATE ARRAY"); + let writer = &cluster.nodes[(ddl_idx + 1) % cluster.nodes.len()]; + for row in 0..SIZE { + let cells: Vec = (0..SIZE) + .map(|col| format!("COORDS ({row}, {col}) VALUES ({}.0)", row * SIZE + col)) + .collect(); + writer + .exec(&format!("INSERT INTO ARRAY czone {}", cells.join(", "))) + .await + .expect("INSERT INTO ARRAY"); + } + writer + .exec("SELECT ARRAY_FLUSH('czone')") + .await + .expect("ARRAY_FLUSH"); + cluster + .wait_for_full_apply_convergence(std::time::Duration::from_secs(10)) + .await; + + let expected = expected_cells(); + assert!( + expected.len() > 10, + "the zone holds cells: {}", + expected.len() + ); + let sql = format!( + "SELECT * FROM ARRAY_SLICE('czone', '{{row: [0, {}]}}', '*', 0, {}, NULL)", + SIZE - 1, + ewkt() + ); + for (idx, node) in cluster.nodes.iter().enumerate() { + let rows = node + .query_named_rows(&sql) + .await + .unwrap_or_else(|e| panic!("node {idx}: {sql}: {e}")); + let got: BTreeSet<(i64, i64)> = rows + .iter() + .map(|r| { + let num = |c: &str| { + r[c].parse::() + .unwrap_or_else(|e| panic!("node {idx}: {c} in {r:?}: {e}")) + as i64 + }; + (num("row"), num("col")) + }) + .collect(); + assert_eq!(rows.len(), got.len(), "node {idx}: no duplicate rows"); + assert_eq!(got, expected, "node {idx}: the zone's cells"); + } + cluster.shutdown().await; +} diff --git a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs index eaf7c5454..d75f1fda7 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/mod.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/mod.rs @@ -53,6 +53,7 @@ mod cluster_array_body_read; mod cluster_array_cell_raft_replication; mod cluster_array_project_elementwise; mod cluster_array_region_agg; +mod cluster_array_slice_zone; mod cluster_array_ws_rpc; mod cluster_backup_remote_cut; mod cluster_backup_restore; diff --git a/nodedb-cluster/src/distributed_array/coordinator/read.rs b/nodedb-cluster/src/distributed_array/coordinator/read.rs index 2b0d09b76..6a98607a3 100644 --- a/nodedb-cluster/src/distributed_array/coordinator/read.rs +++ b/nodedb-cluster/src/distributed_array/coordinator/read.rs @@ -434,6 +434,7 @@ mod tests { system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, txn_id: None, + zone_msgpack: vec![], }; // 3 shards × 2 rows each = 6 merged rows. @@ -475,6 +476,7 @@ mod tests { system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, txn_id: None, + zone_msgpack: vec![], }; let result = coord @@ -660,6 +662,7 @@ mod tests { system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, txn_id: None, + zone_msgpack: vec![], }; // coordinator_limit = 0 → no cutoff → 20 rows. @@ -734,6 +737,7 @@ mod tests { system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, txn_id: None, + zone_msgpack: vec![], }; let result = coord .coord_slice(req, 0, nodedb_types::SystemTimeScope::Current) diff --git a/nodedb-cluster/src/distributed_array/handler.rs b/nodedb-cluster/src/distributed_array/handler.rs index b574a89b2..049043037 100644 --- a/nodedb-cluster/src/distributed_array/handler.rs +++ b/nodedb-cluster/src/distributed_array/handler.rs @@ -416,6 +416,7 @@ mod tests { system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, txn_id: None, + zone_msgpack: vec![], }; zerompk::to_msgpack_vec(&req).unwrap() } @@ -779,6 +780,7 @@ mod tests { system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, txn_id: None, + zone_msgpack: vec![], } } diff --git a/nodedb-cluster/src/distributed_array/wire.rs b/nodedb-cluster/src/distributed_array/wire.rs index 3b3a7616f..bc6209fe1 100644 --- a/nodedb-cluster/src/distributed_array/wire.rs +++ b/nodedb-cluster/src/distributed_array/wire.rs @@ -54,6 +54,9 @@ pub struct ArrayShardSliceReq { /// own leader and stages there, and this read resolves the same leader, /// so the overlay keyed by this id is on the node that serves the read. pub txn_id: Option, + /// Zerompk encoding of `nodedb_array::query::PolygonZone`, forwarded from + /// `ArrayOp::Slice::zone_msgpack`. Empty = no zone. + pub zone_msgpack: Vec, } /// Gather response: shard returns matching rows as opaque msgpack row bytes. diff --git a/nodedb-physical/src/physical_plan/array.rs b/nodedb-physical/src/physical_plan/array.rs index 153b76a86..194aa325f 100644 --- a/nodedb-physical/src/physical_plan/array.rs +++ b/nodedb-physical/src/physical_plan/array.rs @@ -167,6 +167,9 @@ pub enum ArrayOp { /// returned if `valid_from_ms <= vt < valid_until_ms`. `None` = /// no valid-time filter. valid_at_ms: Option, + /// zerompk-encoded `nodedb_array::query::PolygonZone`. Only cells + /// inside it are returned. Empty = no zone. + zone_msgpack: Vec, }, /// Reduce one attribute column. `group_by_dim < 0` means no @@ -334,6 +337,7 @@ mod tests { shard_bucket: None, system_time: SystemTimeScope::AllVersions, valid_at_ms: None, + zone_msgpack: Vec::new(), }; let bytes = zerompk::to_msgpack_vec(&op).unwrap(); let back: ArrayOp = zerompk::from_msgpack(&bytes).unwrap(); @@ -362,6 +366,7 @@ mod tests { shard_bucket: None, system_time: SystemTimeScope::AsOf(1_700_000_000_000), valid_at_ms: Some(1_600_000_000_000), + zone_msgpack: vec![4, 5, 6], }; let bytes = zerompk::to_msgpack_vec(&op).unwrap(); let back: ArrayOp = zerompk::from_msgpack(&bytes).unwrap(); diff --git a/nodedb-physical/src/physical_plan/cluster_array.rs b/nodedb-physical/src/physical_plan/cluster_array.rs index a4e21fcbd..005f91849 100644 --- a/nodedb-physical/src/physical_plan/cluster_array.rs +++ b/nodedb-physical/src/physical_plan/cluster_array.rs @@ -59,6 +59,9 @@ pub enum ClusterArrayOp { system_time: SystemTimeScope, /// Bitemporal valid-time point. `None` = no valid-time filter. valid_at_ms: Option, + /// zerompk-encoded `nodedb_array::query::PolygonZone`, forwarded to + /// every shard. Empty = no zone. + zone_msgpack: Vec, }, /// Fan-out an aggregate across all target vShards and reduce partials. diff --git a/nodedb-physical/src/physical_plan/meta.rs b/nodedb-physical/src/physical_plan/meta.rs index 5532e3b87..f644c69c3 100644 --- a/nodedb-physical/src/physical_plan/meta.rs +++ b/nodedb-physical/src/physical_plan/meta.rs @@ -307,7 +307,7 @@ pub enum MetaOp { cutoff_system_ms: i64, }, - /// Alter the bitemporal retention policy of an array. + /// Alter the bitemporal retention policy or the georef of an array. /// /// All real work (catalog rewrite + registry update) is done on the /// Control Plane before this op is emitted. The Data Plane handler @@ -325,6 +325,9 @@ pub enum MetaOp { array_id: String, audit_retain_ms: Option>, minimum_audit_retain_ms: Option>, + /// Re-encoded `ArraySchema` with the new georef. `None` = schema + /// unchanged. Its content hash must equal the stored `schema_hash`. + schema_msgpack: Option>, }, /// Apply retention to continuous aggregate buckets managed by diff --git a/nodedb-raster/Cargo.toml b/nodedb-raster/Cargo.toml new file mode 100644 index 000000000..b2c2a63c3 --- /dev/null +++ b/nodedb-raster/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "nodedb-raster" +description = "Bounded GeoTIFF decoding for NodeDB array bulk loads" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license = "Apache-2.0" +repository.workspace = true +homepage.workspace = true +documentation = "https://docs.rs/nodedb-raster" +keywords = ["geotiff", "raster", "tiff", "geospatial"] +categories = ["science::geo", "parser-implementations"] + +[dependencies] +nodedb-types = { workspace = true } +thiserror = { workspace = true } +tiff = { workspace = true } diff --git a/nodedb-raster/src/block.rs b/nodedb-raster/src/block.rs new file mode 100644 index 000000000..d941f6ace --- /dev/null +++ b/nodedb-raster/src/block.rs @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Decoded blocks: one typed, row-major sample vector per band. + +use std::mem::size_of; + +use nodedb_types::decode_bounds::checked_decode_capacity; +use tiff::decoder::DecodingResult; + +use crate::error::RasterError; +use crate::layout::BlockWindow; +use crate::sample::{SampleFormatCode, SampleType}; + +/// One band's samples for one block, row-major, `width * height` long. +#[derive(Debug, Clone, PartialEq)] +pub enum BandSamples { + U8(Vec), + U16(Vec), + I16(Vec), + U32(Vec), + F32(Vec), + F64(Vec), +} + +impl BandSamples { + /// Number of samples. + pub fn len(&self) -> usize { + match self { + BandSamples::U8(v) => v.len(), + BandSamples::U16(v) => v.len(), + BandSamples::I16(v) => v.len(), + BandSamples::U32(v) => v.len(), + BandSamples::F32(v) => v.len(), + BandSamples::F64(v) => v.len(), + } + } + + /// True when the band holds no sample. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// The sample type. + pub fn sample_type(&self) -> SampleType { + match self { + BandSamples::U8(_) => SampleType::U8, + BandSamples::U16(_) => SampleType::U16, + BandSamples::I16(_) => SampleType::I16, + BandSamples::U32(_) => SampleType::U32, + BandSamples::F32(_) => SampleType::F32, + BandSamples::F64(_) => SampleType::F64, + } + } + + /// Sample `index` widened to `f64`. Every supported type widens + /// exactly. + pub fn value_f64(&self, index: usize) -> Option { + match self { + BandSamples::U8(v) => v.get(index).map(|s| f64::from(*s)), + BandSamples::U16(v) => v.get(index).map(|s| f64::from(*s)), + BandSamples::I16(v) => v.get(index).map(|s| f64::from(*s)), + BandSamples::U32(v) => v.get(index).map(|s| f64::from(*s)), + BandSamples::F32(v) => v.get(index).map(|s| f64::from(*s)), + BandSamples::F64(v) => v.get(index).copied(), + } + } +} + +/// One decoded block: its window and one sample vector per band. +#[derive(Debug, Clone, PartialEq)] +pub struct RasterBlock { + pub index: u32, + pub window: BlockWindow, + pub bands: Vec, +} + +/// What a decoded chunk must hold. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ChunkShape { + pub index: u32, + pub pixels: usize, + pub bands: usize, + pub limit_bytes: usize, +} + +/// Split a chunky chunk, `bands` samples per pixel, into one vector per +/// band. +pub(crate) fn chunky_bands( + result: DecodingResult, + shape: ChunkShape, +) -> Result, RasterError> { + match result { + DecodingResult::U8(v) => deinterleave(v, shape, BandSamples::U8), + DecodingResult::U16(v) => deinterleave(v, shape, BandSamples::U16), + DecodingResult::I16(v) => deinterleave(v, shape, BandSamples::I16), + DecodingResult::U32(v) => deinterleave(v, shape, BandSamples::U32), + DecodingResult::F32(v) => deinterleave(v, shape, BandSamples::F32), + DecodingResult::F64(v) => deinterleave(v, shape, BandSamples::F64), + other => Err(unsupported(&other)), + } +} + +/// Crop a planar chunk to the block's pixels. A plane's rows are packed at +/// the block width, so the first `pixels` samples are the block. +pub(crate) fn planar_band( + result: DecodingResult, + shape: ChunkShape, +) -> Result { + match result { + DecodingResult::U8(v) => crop(v, shape).map(BandSamples::U8), + DecodingResult::U16(v) => crop(v, shape).map(BandSamples::U16), + DecodingResult::I16(v) => crop(v, shape).map(BandSamples::I16), + DecodingResult::U32(v) => crop(v, shape).map(BandSamples::U32), + DecodingResult::F32(v) => crop(v, shape).map(BandSamples::F32), + DecodingResult::F64(v) => crop(v, shape).map(BandSamples::F64), + other => Err(unsupported(&other)), + } +} + +fn crop(mut samples: Vec, shape: ChunkShape) -> Result, RasterError> { + if samples.len() < shape.pixels { + return Err(RasterError::ShortBlock { + index: shape.index, + found: samples.len(), + expected: shape.pixels, + }); + } + samples.truncate(shape.pixels); + Ok(samples) +} + +fn deinterleave( + samples: Vec, + shape: ChunkShape, + wrap: fn(Vec) -> BandSamples, +) -> Result, RasterError> { + let needed = shape.pixels.saturating_mul(shape.bands); + if samples.len() < needed { + return Err(RasterError::ShortBlock { + index: shape.index, + found: samples.len(), + expected: needed, + }); + } + if shape.bands == 1 { + return crop(samples, shape).map(|band| vec![wrap(band)]); + } + let capacity = checked_decode_capacity( + shape.pixels, + size_of::(), + samples.len(), + shape.bands, + usize::MAX, + shape.limit_bytes, + ) + .ok_or(RasterError::BlockTooLarge { + needed_bytes: (needed as u64).saturating_mul(size_of::() as u64), + limit_bytes: shape.limit_bytes, + })?; + let bands = (0..shape.bands) + .map(|band| { + let mut out = Vec::with_capacity(capacity); + let interleaved = samples.get(band..needed).unwrap_or_default(); + out.extend(interleaved.iter().step_by(shape.bands).copied()); + wrap(out) + }) + .collect(); + Ok(bands) +} + +/// The typed error for a sample type the open-time check refuses. +fn unsupported(result: &DecodingResult) -> RasterError { + let (format, bits) = match result { + DecodingResult::I8(_) => (2, 8), + DecodingResult::I32(_) => (2, 32), + DecodingResult::I64(_) => (2, 64), + DecodingResult::U64(_) => (1, 64), + DecodingResult::F16(_) => (3, 16), + DecodingResult::U8(_) => (1, 8), + DecodingResult::U16(_) => (1, 16), + DecodingResult::I16(_) => (2, 16), + DecodingResult::U32(_) => (1, 32), + DecodingResult::F32(_) => (3, 32), + DecodingResult::F64(_) => (3, 64), + }; + RasterError::UnsupportedSampleType { + format: SampleFormatCode(format), + bits, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn shape(pixels: usize, bands: usize) -> ChunkShape { + ChunkShape { + index: 0, + pixels, + bands, + limit_bytes: 1 << 20, + } + } + + #[test] + fn chunky_samples_split_per_band() { + let result = DecodingResult::U16(vec![1, 10, 100, 2, 20, 200]); + let bands = chunky_bands(result, shape(2, 3)).expect("splits"); + assert_eq!( + bands, + vec![ + BandSamples::U16(vec![1, 2]), + BandSamples::U16(vec![10, 20]), + BandSamples::U16(vec![100, 200]), + ] + ); + } + + #[test] + fn single_band_keeps_its_buffer() { + let result = DecodingResult::F32(vec![1.5, 2.5, 9.0]); + let bands = chunky_bands(result, shape(2, 1)).expect("splits"); + assert_eq!(bands, vec![BandSamples::F32(vec![1.5, 2.5])]); + } + + #[test] + fn planar_plane_is_cropped_to_the_block() { + let result = DecodingResult::I16(vec![-1, -2, -3, -4, 0, 0]); + let band = planar_band(result, shape(4, 1)).expect("crops"); + assert_eq!(band, BandSamples::I16(vec![-1, -2, -3, -4])); + } + + #[test] + fn short_chunks_are_typed_errors() { + assert!(matches!( + chunky_bands(DecodingResult::U8(vec![1, 2, 3]), shape(2, 2)), + Err(RasterError::ShortBlock { + found: 3, + expected: 4, + .. + }) + )); + assert!(matches!( + planar_band(DecodingResult::U8(vec![1]), shape(2, 1)), + Err(RasterError::ShortBlock { .. }) + )); + } + + #[test] + fn deinterleave_respects_the_byte_cap() { + let tight = ChunkShape { + limit_bytes: 3, + ..shape(2, 2) + }; + assert!(matches!( + chunky_bands(DecodingResult::U16(vec![1, 2, 3, 4]), tight), + Err(RasterError::BlockTooLarge { .. }) + )); + } + + #[test] + fn unsupported_variants_are_typed_errors() { + assert!(matches!( + chunky_bands(DecodingResult::I8(vec![1]), shape(1, 1)), + Err(RasterError::UnsupportedSampleType { bits: 8, .. }) + )); + assert!(matches!( + planar_band(DecodingResult::U64(vec![1]), shape(1, 1)), + Err(RasterError::UnsupportedSampleType { bits: 64, .. }) + )); + } + + #[test] + fn values_widen_to_f64() { + let band = BandSamples::I16(vec![-7, 3]); + assert_eq!(band.value_f64(0), Some(-7.0)); + assert_eq!(band.value_f64(2), None); + assert_eq!(band.sample_type(), SampleType::I16); + assert_eq!(band.len(), 2); + } +} diff --git a/nodedb-raster/src/crs.rs b/nodedb-raster/src/crs.rs new file mode 100644 index 000000000..d9218bb67 --- /dev/null +++ b/nodedb-raster/src/crs.rs @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The raster CRS, from the GeoKey directory. +//! +//! A projected model (1) reads `ProjectedCRSGeoKey` (3072), a geographic +//! model (2) reads `GeodeticCRSGeoKey` (2048). With no model key, 3072 wins +//! over 2048. The EPSG code then maps through [`Srid::from_code`]. + +use nodedb_types::geometry::Srid; + +use crate::error::RasterError; +use crate::geokeys::{GEOGRAPHIC_TYPE, GT_MODEL_TYPE, GeoKeyDirectory, PROJECTED_CS_TYPE}; + +const MODEL_PROJECTED: u16 = 1; +const MODEL_GEOGRAPHIC: u16 = 2; +/// GeoKey value of a CRS defined by parameters, not by an EPSG code. +const USER_DEFINED: u16 = 32767; +/// GeoKey value of an undefined CRS. +const UNDEFINED: u16 = 0; + +/// The SRID the GeoKeys name, or `None` when they name no CRS. +pub fn srid_from_geokeys(keys: Option<&GeoKeyDirectory>) -> Result, RasterError> { + let Some(keys) = keys else { + return Ok(None); + }; + let key = match keys.short(GT_MODEL_TYPE) { + Some(MODEL_PROJECTED) => PROJECTED_CS_TYPE, + Some(MODEL_GEOGRAPHIC) => GEOGRAPHIC_TYPE, + Some(model) => return Err(RasterError::UnsupportedModelType { model }), + None if keys.short(PROJECTED_CS_TYPE).is_some() => PROJECTED_CS_TYPE, + None if keys.short(GEOGRAPHIC_TYPE).is_some() => GEOGRAPHIC_TYPE, + None => return Ok(None), + }; + let model = keys.short(GT_MODEL_TYPE).unwrap_or(UNDEFINED); + match keys.short(key) { + None | Some(UNDEFINED) => Err(RasterError::MissingCrsKey { model, key }), + Some(USER_DEFINED) => Err(RasterError::UserDefinedCrs { key }), + Some(code) => Ok(Some(Srid::from_code(i64::from(code))?)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn keys(entries: &[(u16, u16)]) -> GeoKeyDirectory { + let count = u16::try_from(entries.len()).expect("few keys"); + let mut dir = vec![1, 1, 0, count]; + for (key, value) in entries { + dir.extend([*key, 0, 1, *value]); + } + GeoKeyDirectory::parse(&dir, None, None).expect("valid directory") + } + + #[test] + fn no_directory_is_no_crs() { + assert_eq!(srid_from_geokeys(None).ok(), Some(None)); + } + + #[test] + fn projected_model_reads_3072() { + let dir = keys(&[(GT_MODEL_TYPE, 1), (PROJECTED_CS_TYPE, 32647)]); + assert_eq!(srid_from_geokeys(Some(&dir)).ok(), Some(Some(Srid::Utm47N))); + } + + #[test] + fn geographic_model_reads_2048() { + let dir = keys(&[(GT_MODEL_TYPE, 2), (GEOGRAPHIC_TYPE, 4326)]); + assert_eq!(srid_from_geokeys(Some(&dir)).ok(), Some(Some(Srid::Wgs84))); + } + + #[test] + fn model_key_picks_the_crs_key() { + let dir = keys(&[ + (GT_MODEL_TYPE, 2), + (GEOGRAPHIC_TYPE, 4326), + (PROJECTED_CS_TYPE, 32648), + ]); + assert_eq!(srid_from_geokeys(Some(&dir)).ok(), Some(Some(Srid::Wgs84))); + } + + #[test] + fn without_a_model_key_3072_wins() { + let dir = keys(&[(GEOGRAPHIC_TYPE, 4326), (PROJECTED_CS_TYPE, 3375)]); + assert_eq!( + srid_from_geokeys(Some(&dir)).ok(), + Some(Some(Srid::PeninsularRso)) + ); + let none = keys(&[(1025, 1)]); + assert_eq!(srid_from_geokeys(Some(&none)).ok(), Some(None)); + } + + #[test] + fn unmappable_crs_are_typed_errors() { + let geocentric = keys(&[(GT_MODEL_TYPE, 3)]); + assert!(matches!( + srid_from_geokeys(Some(&geocentric)), + Err(RasterError::UnsupportedModelType { model: 3 }) + )); + let missing = keys(&[(GT_MODEL_TYPE, 1)]); + assert!(matches!( + srid_from_geokeys(Some(&missing)), + Err(RasterError::MissingCrsKey { + model: 1, + key: 3072 + }) + )); + let user = keys(&[(GT_MODEL_TYPE, 1), (PROJECTED_CS_TYPE, 32767)]); + assert!(matches!( + srid_from_geokeys(Some(&user)), + Err(RasterError::UserDefinedCrs { key: 3072 }) + )); + let other = keys(&[(GT_MODEL_TYPE, 1), (PROJECTED_CS_TYPE, 32633)]); + assert!(matches!( + srid_from_geokeys(Some(&other)), + Err(RasterError::UnsupportedCrs(_)) + )); + } +} diff --git a/nodedb-raster/src/error.rs b/nodedb-raster/src/error.rs new file mode 100644 index 000000000..db090436d --- /dev/null +++ b/nodedb-raster/src/error.rs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Typed errors of the GeoTIFF reader. + +use std::io; + +use nodedb_types::geometry::UnsupportedSrid; +use tiff::TiffError; + +use crate::geokeys::GeoKeyError; +use crate::sample::SampleFormatCode; + +/// Every way opening or decoding a GeoTIFF can fail. +#[derive(Debug, thiserror::Error)] +pub enum RasterError { + /// The underlying reader failed for a reason outside the file's bytes. + #[error("GeoTIFF read failed: {0}")] + Io(#[source] io::Error), + + /// The bytes do not form a valid TIFF: truncated, corrupt, or + /// self-contradictory. + #[error("malformed TIFF: {0}")] + Malformed(#[source] TiffError), + + /// The TIFF is valid, but it uses a feature the decoder does not read. + #[error("unsupported TIFF feature: {0}")] + UnsupportedTiff(#[source] TiffError), + + /// One block decodes to more bytes than the caller allows. + #[error( + "GeoTIFF block needs {needed_bytes} decoded bytes, over the {limit_bytes}-byte limit; \ + rewrite the file with smaller strips or tiles" + )] + BlockTooLarge { + needed_bytes: u64, + limit_bytes: usize, + }, + + /// An IFD 0 tag value ends past the end of the file or exceeds the + /// tag value cap. + #[error( + "GeoTIFF tag {tag} declares a {bytes}-byte value, past the {file_len}-byte file \ + or over the tag value cap" + )] + TagValueOutOfBounds { tag: u16, bytes: u64, file_len: u64 }, + + /// The tiff decoder hit its own buffer limit. + #[error("GeoTIFF decode exceeds the {limit_bytes}-byte buffer limit")] + DecodeLimitExceeded { limit_bytes: usize }, + + /// The compression method is not deflate, LZW, PackBits, or none. + #[error( + "GeoTIFF compression code {code} is not supported; \ + supported: none (1), LZW (5), deflate (8, 32946), PackBits (32773)" + )] + UnsupportedCompression { code: u16 }, + + /// The sample format and width are outside u8, u16, i16, u32, f32, f64. + #[error( + "GeoTIFF samples of format {format} with {bits} bits are not supported; \ + supported: u8, u16, i16, u32, f32, f64" + )] + UnsupportedSampleType { format: SampleFormatCode, bits: u8 }, + + /// The photometric interpretation transforms samples on decode, or + /// stores palette indexes. + #[error( + "GeoTIFF photometric interpretation {code} is not supported; \ + supported: BlackIsZero (1), RGB (2), CMYK (5), YCbCr (6)" + )] + UnsupportedPhotometric { code: u16 }, + + /// A chunky pixel decodes to fewer samples than the file stores. + #[error( + "GeoTIFF stores {samples} samples per pixel, but its color type decodes {decoded}; \ + write the extra bands as BlackIsZero multiband" + )] + UnsupportedBandLayout { samples: u16, decoded: u16 }, + + /// IFD 0 is an overview, so the full-resolution image is elsewhere. + #[error( + "GeoTIFF IFD 0 is a reduced-resolution image; \ + the full-resolution image must be the first IFD" + )] + ReducedResolutionFirstImage, + + /// The GeoKey directory does not parse. + #[error("malformed GeoKey directory: {0}")] + GeoKeys(#[from] GeoKeyError), + + /// A georeference tag holds the wrong number of values. + #[error("GeoTIFF tag {tag} holds {found} values, expected {expected}")] + GeoTagLength { + tag: &'static str, + found: usize, + expected: &'static str, + }, + + /// Tiepoints without a pixel scale or a transformation are ground + /// control points. + #[error( + "GeoTIFF georeferences by ground control points; \ + warp it to an affine grid first" + )] + GroundControlPoints, + + /// A pixel scale without a tiepoint places no pixel in the world. + #[error("GeoTIFF has ModelPixelScaleTag but no ModelTiepointTag")] + PixelScaleWithoutTiepoint, + + /// The model transformation is not a 2D affine transform. + #[error( + "GeoTIFF ModelTransformationTag is not a 2D affine transform: \ + the z terms must be 0 and the last row 0 0 0 1" + )] + NonAffineTransformation, + + /// The geotransform holds a non-finite term or maps the plane to a line. + #[error("GeoTIFF geotransform {transform:?} is degenerate")] + DegenerateTransform { transform: [f64; 6] }, + + /// The GeoKey model type is neither projected (1) nor geographic (2). + #[error( + "GeoTIFF model type {model} is not supported; supported: projected (1), geographic (2)" + )] + UnsupportedModelType { model: u16 }, + + /// The model type names a CRS key the directory does not hold. + #[error("GeoTIFF model type {model} needs GeoKey {key}, which is absent")] + MissingCrsKey { model: u16, key: u16 }, + + /// The CRS is user-defined, so no EPSG code names it. + #[error("GeoTIFF CRS is user-defined (GeoKey {key} = 32767); only EPSG codes are supported")] + UserDefinedCrs { key: u16 }, + + /// The CRS EPSG code is outside the supported set. + #[error("GeoTIFF CRS: {0}")] + UnsupportedCrs(#[from] UnsupportedSrid), + + /// GDAL_NODATA is not a number. + #[error("GeoTIFF GDAL_NODATA '{text}' is not a number")] + InvalidNodata { text: String }, + + /// The caller asked for a block the image does not have. + #[error("GeoTIFF block {index} is out of range; the image has {count} blocks")] + BlockOutOfRange { index: u32, count: u32 }, + + /// The decoder returned fewer samples than the block holds. + #[error("GeoTIFF block {index} decoded {found} samples, expected at least {expected}")] + ShortBlock { + index: u32, + found: usize, + expected: usize, + }, +} + +impl RasterError { + /// Classify a tiff decoder error. `limit_bytes` is the buffer cap the + /// decoder ran under. + pub(crate) fn from_tiff(error: TiffError, limit_bytes: usize) -> Self { + match error { + TiffError::IoError(io) => RasterError::from(io), + TiffError::UnsupportedError(_) => RasterError::UnsupportedTiff(error), + TiffError::LimitsExceeded => RasterError::DecodeLimitExceeded { limit_bytes }, + TiffError::FormatError(_) | TiffError::IntSizeError | TiffError::UsageError(_) => { + RasterError::Malformed(error) + } + } + } +} + +impl From for RasterError { + /// A short or undecodable stream is a malformed file. Every other kind + /// is a reader error. + fn from(error: io::Error) -> Self { + match error.kind() { + io::ErrorKind::UnexpectedEof + | io::ErrorKind::InvalidData + | io::ErrorKind::InvalidInput => RasterError::Malformed(TiffError::IoError(error)), + _ => RasterError::Io(error), + } + } +} diff --git a/nodedb-raster/src/geokeys.rs b/nodedb-raster/src/geokeys.rs new file mode 100644 index 000000000..7d85a8f91 --- /dev/null +++ b/nodedb-raster/src/geokeys.rs @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! GeoKey directory parsing (OGC GeoTIFF 1.1, Requirements Class +//! GeoKeyDirectoryTag). +//! +//! The directory is a `u16` array: a 4-short header, then 4 shorts per +//! key. A key value lives inline (location 0), in the directory array +//! itself (34735), in GeoDoubleParamsTag (34736), or in GeoAsciiParamsTag +//! (34737). Every reference is bounds-checked against the array it names. + +use std::mem::size_of; + +use nodedb_types::decode_bounds::checked_decode_capacity; + +/// `GTModelTypeGeoKey`: 1 projected, 2 geographic, 3 geocentric. +pub const GT_MODEL_TYPE: u16 = 1024; +/// `GTRasterTypeGeoKey`: 1 PixelIsArea, 2 PixelIsPoint. +pub const GT_RASTER_TYPE: u16 = 1025; +/// `GeodeticCRSGeoKey` (`GeographicTypeGeoKey` in GeoTIFF 1.0). +pub const GEOGRAPHIC_TYPE: u16 = 2048; +/// `ProjectedCRSGeoKey` (`ProjectedCSTypeGeoKey` in GeoTIFF 1.0). +pub const PROJECTED_CS_TYPE: u16 = 3072; + +const DIRECTORY_TAG: u16 = 34735; +const DOUBLE_PARAMS_TAG: u16 = 34736; +const ASCII_PARAMS_TAG: u16 = 34737; +const HEADER_SHORTS: usize = 4; +const ENTRY_SHORTS: usize = 4; +const DIRECTORY_VERSION: u16 = 1; +/// Upper bound on the bytes of the parsed key list. +const MAX_KEY_LIST_BYTES: usize = 4 * 1024 * 1024; + +/// Every way a GeoKey directory can be malformed. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum GeoKeyError { + #[error("the directory holds {len} shorts, fewer than its 4-short header")] + TooShort { len: usize }, + + #[error("directory version {version} is not 1")] + Version { version: u16 }, + + #[error("the header declares {declared} keys, but the directory holds {available}")] + KeyCount { declared: u16, available: usize }, + + #[error("key {key} follows key {previous}; keys must be strictly ascending")] + UnsortedKey { key: u16, previous: u16 }, + + #[error("key {key} has an inline value with count {count}; an inline value has count 1")] + InlineCount { key: u16, count: u16 }, + + #[error("key {key} stores its value in tag {location}, which the file lacks")] + MissingParams { key: u16, location: u16 }, + + #[error("key {key} stores its value in unknown tag {location}")] + UnknownLocation { key: u16, location: u16 }, + + #[error( + "key {key} reads {count} values at offset {offset} of tag {location}, \ + which holds {len}" + )] + OutOfBounds { + key: u16, + location: u16, + offset: u16, + count: u16, + len: usize, + }, + + #[error("key {key} ASCII value is not UTF-8")] + AsciiEncoding { key: u16 }, +} + +/// One key value, in the type of the array it came from. +#[derive(Debug, Clone, PartialEq)] +pub enum GeoKeyValue { + Short(Vec), + Double(Vec), + Ascii(String), +} + +/// A parsed GeoKey directory, keys strictly ascending. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct GeoKeyDirectory { + keys: Vec<(u16, GeoKeyValue)>, +} + +impl GeoKeyDirectory { + /// Parse `directory` (GeoKeyDirectoryTag) with its optional double and + /// ASCII parameter arrays. + pub fn parse( + directory: &[u16], + doubles: Option<&[f64]>, + ascii: Option<&str>, + ) -> Result { + let (header, body) = match directory.split_at_checked(HEADER_SHORTS) { + Some(split) => split, + None => { + return Err(GeoKeyError::TooShort { + len: directory.len(), + }); + } + }; + if header[0] != DIRECTORY_VERSION { + return Err(GeoKeyError::Version { version: header[0] }); + } + let declared = header[3]; + let available = body.len() / ENTRY_SHORTS; + let capacity = checked_decode_capacity( + usize::from(declared), + size_of::<(u16, GeoKeyValue)>(), + body.len(), + ENTRY_SHORTS, + usize::from(u16::MAX), + MAX_KEY_LIST_BYTES, + ) + .ok_or(GeoKeyError::KeyCount { + declared, + available, + })?; + let mut keys: Vec<(u16, GeoKeyValue)> = Vec::with_capacity(capacity); + for &[key, location, count, offset] in + body.as_chunks::().0.iter().take(capacity) + { + if let Some((previous, _)) = keys.last() + && *previous >= key + { + return Err(GeoKeyError::UnsortedKey { + key, + previous: *previous, + }); + } + let reference = Reference { + key, + location, + offset, + count, + }; + let value = match location { + 0 => { + if count != 1 { + return Err(GeoKeyError::InlineCount { key, count }); + } + GeoKeyValue::Short(vec![offset]) + } + DIRECTORY_TAG => GeoKeyValue::Short(reference.slice(directory)?.to_vec()), + DOUBLE_PARAMS_TAG => { + let doubles = doubles.ok_or(GeoKeyError::MissingParams { key, location })?; + GeoKeyValue::Double(reference.slice(doubles)?.to_vec()) + } + ASCII_PARAMS_TAG => { + let ascii = ascii.ok_or(GeoKeyError::MissingParams { key, location })?; + let bytes = reference.slice(ascii.as_bytes())?; + let text = std::str::from_utf8(bytes) + .map_err(|_| GeoKeyError::AsciiEncoding { key })?; + // Each ASCII value ends with a `|` separator. + let text = text.strip_suffix('|').unwrap_or(text); + GeoKeyValue::Ascii(text.to_owned()) + } + _ => return Err(GeoKeyError::UnknownLocation { key, location }), + }; + keys.push((key, value)); + } + Ok(Self { keys }) + } + + /// The value of `key`. + pub fn get(&self, key: u16) -> Option<&GeoKeyValue> { + self.keys + .binary_search_by_key(&key, |(seen, _)| *seen) + .ok() + .and_then(|index| self.keys.get(index)) + .map(|(_, value)| value) + } + + /// The first short of `key`, when `key` holds shorts. + pub fn short(&self, key: u16) -> Option { + match self.get(key)? { + GeoKeyValue::Short(values) => values.first().copied(), + GeoKeyValue::Double(_) | GeoKeyValue::Ascii(_) => None, + } + } + + /// The number of keys. + pub fn len(&self) -> usize { + self.keys.len() + } + + /// True when the directory holds no key. + pub fn is_empty(&self) -> bool { + self.keys.is_empty() + } +} + +/// Where one key's value sits inside a parameter array. +struct Reference { + key: u16, + location: u16, + offset: u16, + count: u16, +} + +impl Reference { + /// The `count` values at `offset` of `values`, bounds-checked. + fn slice<'v, T>(&self, values: &'v [T]) -> Result<&'v [T], GeoKeyError> { + let start = usize::from(self.offset); + let end = start + usize::from(self.count); + values.get(start..end).ok_or(GeoKeyError::OutOfBounds { + key: self.key, + location: self.location, + offset: self.offset, + count: self.count, + len: values.len(), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn header(keys: u16) -> Vec { + vec![1, 1, 0, keys] + } + + #[test] + fn inline_keys_parse() { + let mut dir = header(3); + dir.extend([GT_MODEL_TYPE, 0, 1, 1]); + dir.extend([GT_RASTER_TYPE, 0, 1, 2]); + dir.extend([PROJECTED_CS_TYPE, 0, 1, 32647]); + let parsed = GeoKeyDirectory::parse(&dir, None, None).expect("parses"); + assert_eq!(parsed.len(), 3); + assert_eq!(parsed.short(GT_MODEL_TYPE), Some(1)); + assert_eq!(parsed.short(GT_RASTER_TYPE), Some(2)); + assert_eq!(parsed.short(PROJECTED_CS_TYPE), Some(32647)); + } + + #[test] + fn parameter_arrays_are_read_by_offset() { + let mut dir = header(2); + dir.extend([1026, ASCII_PARAMS_TAG, 6, 0]); + dir.extend([2057, DOUBLE_PARAMS_TAG, 2, 1]); + let doubles = [0.0, 6_378_137.0, 298.257]; + let parsed = + GeoKeyDirectory::parse(&dir, Some(&doubles), Some("UTM47|other|")).expect("parses"); + assert_eq!( + parsed.get(2057), + Some(&GeoKeyValue::Double(vec![6_378_137.0, 298.257])) + ); + assert_eq!( + parsed.get(1026), + Some(&GeoKeyValue::Ascii("UTM47".to_owned())) + ); + } + + #[test] + fn directory_shorts_are_read_from_the_directory() { + let mut dir = header(1); + dir.extend([4000, DIRECTORY_TAG, 2, 8]); + dir.extend([7, 9]); + let parsed = GeoKeyDirectory::parse(&dir, None, None).expect("parses"); + assert_eq!(parsed.get(4000), Some(&GeoKeyValue::Short(vec![7, 9]))); + } + + #[test] + fn key_count_is_bounded_by_the_array() { + let mut dir = header(5); + dir.extend([GT_MODEL_TYPE, 0, 1, 1]); + assert_eq!( + GeoKeyDirectory::parse(&dir, None, None), + Err(GeoKeyError::KeyCount { + declared: 5, + available: 1 + }) + ); + } + + #[test] + fn malformed_directories_are_typed_errors() { + assert!(matches!( + GeoKeyDirectory::parse(&[1, 1], None, None), + Err(GeoKeyError::TooShort { len: 2 }) + )); + assert!(matches!( + GeoKeyDirectory::parse(&[2, 1, 0, 0], None, None), + Err(GeoKeyError::Version { version: 2 }) + )); + + let mut dup = header(2); + dup.extend([GT_MODEL_TYPE, 0, 1, 1, GT_MODEL_TYPE, 0, 1, 2]); + assert!(matches!( + GeoKeyDirectory::parse(&dup, None, None), + Err(GeoKeyError::UnsortedKey { .. }) + )); + + let mut unsorted = header(2); + unsorted.extend([GT_RASTER_TYPE, 0, 1, 1, GT_MODEL_TYPE, 0, 1, 1]); + assert!(matches!( + GeoKeyDirectory::parse(&unsorted, None, None), + Err(GeoKeyError::UnsortedKey { .. }) + )); + + let mut inline = header(1); + inline.extend([GT_MODEL_TYPE, 0, 2, 1]); + assert!(matches!( + GeoKeyDirectory::parse(&inline, None, None), + Err(GeoKeyError::InlineCount { .. }) + )); + + let mut missing = header(1); + missing.extend([2057, DOUBLE_PARAMS_TAG, 1, 0]); + assert!(matches!( + GeoKeyDirectory::parse(&missing, None, None), + Err(GeoKeyError::MissingParams { .. }) + )); + + let mut past = header(1); + past.extend([2057, DOUBLE_PARAMS_TAG, 4, 2]); + assert!(matches!( + GeoKeyDirectory::parse(&past, Some(&[1.0, 2.0, 3.0]), None), + Err(GeoKeyError::OutOfBounds { .. }) + )); + + let mut unknown = header(1); + unknown.extend([2057, 999, 1, 0]); + assert!(matches!( + GeoKeyDirectory::parse(&unknown, None, None), + Err(GeoKeyError::UnknownLocation { .. }) + )); + } + + #[test] + fn ascii_slice_inside_a_multibyte_char_is_an_error() { + let mut dir = header(1); + dir.extend([1026, ASCII_PARAMS_TAG, 1, 0]); + assert!(matches!( + GeoKeyDirectory::parse(&dir, None, Some("é|")), + Err(GeoKeyError::AsciiEncoding { .. }) + )); + } +} diff --git a/nodedb-raster/src/layout.rs b/nodedb-raster/src/layout.rs new file mode 100644 index 000000000..7db3bf692 --- /dev/null +++ b/nodedb-raster/src/layout.rs @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The block grid of IFD 0: image size, band layout, and strip or tile +//! geometry, read once at open. + +use std::io::{Read, Seek}; + +use tiff::decoder::{ChunkType, Decoder}; +use tiff::tags::{SampleFormat, Tag}; + +use crate::error::RasterError; +use crate::sample::SampleType; + +/// `PlanarConfiguration` value of band-separate storage. +const PLANAR_SEPARATE: u16 = 2; +/// Photometric interpretations whose samples decode unchanged: +/// BlackIsZero, RGB, CMYK, YCbCr. +const RAW_PHOTOMETRICS: [u16; 4] = [1, 2, 5, 6]; + +/// How a raster stores its blocks. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BlockKind { + /// Full-width strips of rows. + Strips, + /// Rectangular tiles. + Tiles, +} + +/// How a raster stores its bands. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Interleave { + /// All bands of a pixel together (`PlanarConfiguration` 1). + Chunky, + /// One plane per band (`PlanarConfiguration` 2). + Planar, +} + +/// The pixel rectangle one block covers. Edge blocks are cropped to the +/// image. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BlockWindow { + pub col_off: u32, + pub row_off: u32, + pub width: u32, + pub height: u32, +} + +impl BlockWindow { + /// Pixels in the window. + pub fn pixels(&self) -> usize { + self.width as usize * self.height as usize + } +} + +/// The geometry every block read derives from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RasterLayout { + pub width: u32, + pub height: u32, + pub bands: u16, + pub sample_type: SampleType, + pub kind: BlockKind, + pub interleave: Interleave, + /// Block width as stored. A tile can extend past the image edge. + pub block_width: u32, + /// Block height as stored. A strip height is clamped to the image. + pub block_height: u32, + pub blocks_across: u32, + pub blocks_down: u32, +} + +impl RasterLayout { + /// Read the layout of the decoder's current image. + pub(crate) fn read( + decoder: &mut Decoder, + limit_bytes: usize, + ) -> Result { + let tiff = |e| RasterError::from_tiff(e, limit_bytes); + let photometric = decoder + .find_tag_unsigned::(Tag::PhotometricInterpretation) + .map_err(tiff)? + .unwrap_or(1); + if !RAW_PHOTOMETRICS.contains(&photometric) { + return Err(RasterError::UnsupportedPhotometric { code: photometric }); + } + let (width, height) = decoder.dimensions().map_err(tiff)?; + let color = decoder.colortype().map_err(tiff)?; + let bands = decoder + .find_tag_unsigned::(Tag::SamplesPerPixel) + .map_err(tiff)? + .unwrap_or(1); + let format = decoder + .find_tag_unsigned_vec::(Tag::SampleFormat) + .map_err(tiff)? + .and_then(|formats| formats.first().copied()) + .map_or(SampleFormat::Uint, SampleFormat::from_u16_exhaustive); + let sample_type = SampleType::from_tiff(format, color.bit_depth())?; + let interleave = match decoder + .find_tag_unsigned::(Tag::PlanarConfiguration) + .map_err(tiff)? + { + Some(PLANAR_SEPARATE) => Interleave::Planar, + _ => Interleave::Chunky, + }; + // A chunky pixel decodes to the color type's samples. Fewer than the + // file stores drops bands. + if interleave == Interleave::Chunky && color.num_samples() != bands { + return Err(RasterError::UnsupportedBandLayout { + samples: bands, + decoded: color.num_samples(), + }); + } + // The tiff decoder refuses a zero image size, tile size, and + // RowsPerStrip, so every block dimension here is at least 1. + let (stored_width, stored_height) = decoder.chunk_dimensions(); + let (kind, block_width, block_height) = match decoder.get_chunk_type() { + ChunkType::Strip => (BlockKind::Strips, width, stored_height.min(height)), + ChunkType::Tile => (BlockKind::Tiles, stored_width, stored_height), + }; + Ok(Self { + width, + height, + bands, + sample_type, + kind, + interleave, + block_width, + block_height, + blocks_across: width.div_ceil(block_width), + blocks_down: height.div_ceil(block_height), + }) + } + + /// Blocks per band plane. + pub fn block_count(&self) -> u32 { + // The tiff decoder checked that the chunk offsets, a u32-countable + // list, hold this many entries per plane. + self.blocks_across.saturating_mul(self.blocks_down) + } + + /// Decoded bytes of the largest block, across all bands. + pub fn block_bytes(&self) -> u64 { + u64::from(self.block_width) + * u64::from(self.block_height) + * u64::from(self.bands) + * self.sample_type.byte_len() as u64 + } + + /// The window of block `index`, cropped to the image. + pub fn window(&self, index: u32) -> Result { + let count = self.block_count(); + if index >= count { + return Err(RasterError::BlockOutOfRange { index, count }); + } + let col_off = (index % self.blocks_across) * self.block_width; + let row_off = (index / self.blocks_across) * self.block_height; + Ok(BlockWindow { + col_off, + row_off, + width: self.block_width.min(self.width - col_off), + height: self.block_height.min(self.height - row_off), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn layout(kind: BlockKind, block: (u32, u32), image: (u32, u32)) -> RasterLayout { + RasterLayout { + width: image.0, + height: image.1, + bands: 2, + sample_type: SampleType::U16, + kind, + interleave: Interleave::Chunky, + block_width: block.0, + block_height: block.1, + blocks_across: image.0.div_ceil(block.0), + blocks_down: image.1.div_ceil(block.1), + } + } + + #[test] + fn edge_tiles_are_cropped() { + let grid = layout(BlockKind::Tiles, (16, 16), (40, 24)); + assert_eq!(grid.block_count(), 6); + let last = grid.window(5).expect("in range"); + assert_eq!( + last, + BlockWindow { + col_off: 32, + row_off: 16, + width: 8, + height: 8 + } + ); + let middle = grid.window(1).expect("in range"); + assert_eq!((middle.col_off, middle.width, middle.height), (16, 16, 16)); + assert_eq!(grid.block_bytes(), 16 * 16 * 2 * 2); + } + + #[test] + fn strips_span_the_width() { + let grid = layout(BlockKind::Strips, (10, 4), (10, 9)); + assert_eq!(grid.block_count(), 3); + let last = grid.window(2).expect("in range"); + assert_eq!((last.row_off, last.width, last.height), (8, 10, 1)); + } + + #[test] + fn out_of_range_block_is_a_typed_error() { + let grid = layout(BlockKind::Strips, (10, 4), (10, 9)); + assert!(matches!( + grid.window(3), + Err(RasterError::BlockOutOfRange { index: 3, count: 3 }) + )); + } +} diff --git a/nodedb-raster/src/lib.rs b/nodedb-raster/src/lib.rs new file mode 100644 index 000000000..04a966306 --- /dev/null +++ b/nodedb-raster/src/lib.rs @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Bounded GeoTIFF decoding for NodeDB array bulk loads. +//! +//! [`GeoTiffReader`] opens IFD 0 of a GeoTIFF and decodes it one strip or +//! tile at a time into typed per-band samples, under a caller-given byte +//! cap. It reads the georeference (geotransform, CRS, `GDAL_NODATA`) from +//! the GeoTIFF tags. + +pub mod block; +pub mod crs; +pub mod error; +pub mod geokeys; +pub mod layout; +pub mod limits; +pub mod nodata; +mod preflight; +pub mod reader; +pub mod sample; +pub mod transform; + +pub use block::{BandSamples, RasterBlock}; +pub use crs::srid_from_geokeys; +pub use error::RasterError; +pub use geokeys::{GeoKeyDirectory, GeoKeyError, GeoKeyValue}; +pub use layout::{BlockKind, BlockWindow, Interleave, RasterLayout}; +pub use limits::RasterLimits; +pub use nodata::parse_gdal_nodata; +pub use reader::GeoTiffReader; +pub use sample::{SampleFormatCode, SampleType}; +pub use transform::{GeoTransform, ModelTags}; diff --git a/nodedb-raster/src/limits.rs b/nodedb-raster/src/limits.rs new file mode 100644 index 000000000..0d5c5856d --- /dev/null +++ b/nodedb-raster/src/limits.rs @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Memory caps the reader decodes under. + +use tiff::decoder::Limits; + +/// Largest out-of-line IFD 0 tag value: strip offsets, GeoKeys, GDAL_NODATA. +pub(crate) const MAX_TAG_VALUE_BYTES: usize = 1024 * 1024; + +/// Caps on what one decode allocates. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RasterLimits { + /// Most bytes one block decodes to, across all its bands. The same cap + /// bounds one compressed chunk read from the file. + pub block_bytes: usize, +} + +impl RasterLimits { + /// Limits with `block_bytes` as the per-block decoded byte cap. + pub const fn new(block_bytes: usize) -> Self { + Self { block_bytes } + } + + /// The tiff decoder limits these caps imply. + pub(crate) fn tiff_limits(self) -> Limits { + // `Limits` is non-exhaustive: start from a constructor, then set + // every field this tiff version has. + let mut limits = Limits::unlimited(); + limits.decoding_buffer_size = self.block_bytes; + limits.intermediate_buffer_size = self.block_bytes; + limits.ifd_value_size = MAX_TAG_VALUE_BYTES; + limits + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn block_cap_bounds_decode_and_compressed_buffers() { + let limits = RasterLimits::new(4096).tiff_limits(); + assert_eq!(limits.decoding_buffer_size, 4096); + assert_eq!(limits.intermediate_buffer_size, 4096); + assert_eq!(limits.ifd_value_size, MAX_TAG_VALUE_BYTES); + } +} diff --git a/nodedb-raster/src/nodata.rs b/nodedb-raster/src/nodata.rs new file mode 100644 index 000000000..572e08783 --- /dev/null +++ b/nodedb-raster/src/nodata.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The `GDAL_NODATA` tag (42113): one ASCII number shared by every band. + +use crate::error::RasterError; + +/// Parse a `GDAL_NODATA` value. NUL and space padding is trimmed, and +/// `nan` in any case is NaN. +pub fn parse_gdal_nodata(text: &str) -> Result { + let trimmed = text.trim_matches(|c: char| c == '\0' || c == ' '); + if trimmed.eq_ignore_ascii_case("nan") { + return Ok(f64::NAN); + } + trimmed + .parse::() + .map_err(|_| RasterError::InvalidNodata { + text: trimmed.to_owned(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn integers_and_floats_parse() { + assert_eq!(parse_gdal_nodata("-9999").ok(), Some(-9999.0)); + assert_eq!(parse_gdal_nodata("0").ok(), Some(0.0)); + assert_eq!( + parse_gdal_nodata("-3.4028234663852886e+38").ok(), + Some(f64::from(f32::MIN)) + ); + } + + #[test] + fn padding_is_trimmed() { + assert_eq!(parse_gdal_nodata(" 255\0\0").ok(), Some(255.0)); + assert_eq!(parse_gdal_nodata("\0 -1 \0").ok(), Some(-1.0)); + } + + #[test] + fn nan_in_any_case_is_nan() { + for text in ["nan", "NaN", "NAN", " nan\0"] { + assert!(parse_gdal_nodata(text).is_ok_and(f64::is_nan), "{text}"); + } + } + + #[test] + fn non_numbers_are_refused() { + for text in ["", "\0", "abc", "1,5", "12abc"] { + assert!( + matches!( + parse_gdal_nodata(text), + Err(RasterError::InvalidNodata { .. }) + ), + "{text}" + ); + } + } +} diff --git a/nodedb-raster/src/preflight.rs b/nodedb-raster/src/preflight.rs new file mode 100644 index 000000000..dbdedd83f --- /dev/null +++ b/nodedb-raster/src/preflight.rs @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! A bounds pass over the IFD 0 entry table before the tiff decoder reads +//! it. +//! +//! The tiff decoder reads IFD 0 under its default limits, before a caller +//! can set its own. It reserves a tag's value list from the entry's count. +//! This pass refuses any out-of-line value that is larger than +//! [`MAX_TAG_VALUE_BYTES`] or ends past the end of the file. So no tag +//! value allocation exceeds the file's own size. +//! +//! A header or entry table the pass cannot read is left to the tiff +//! decoder, which refuses it without a large allocation. + +use std::io::{self, Read, Seek, SeekFrom}; + +use crate::error::RasterError; +use crate::limits::MAX_TAG_VALUE_BYTES; + +const CLASSIC_MAGIC: u16 = 42; +const BIG_MAGIC: u16 = 43; + +#[derive(Clone, Copy)] +enum Order { + Little, + Big, +} + +/// The fixed-width entry layout of classic TIFF or BigTIFF. +#[derive(Clone, Copy)] +struct Format { + order: Order, + big: bool, +} + +impl Format { + fn entry_len(self) -> u64 { + if self.big { 20 } else { 12 } + } + + fn inline_len(self) -> u64 { + if self.big { 8 } else { 4 } + } +} + +/// Check every out-of-line IFD 0 value of `reader`, then rewind it. +pub(crate) fn check_ifd0(reader: &mut R) -> Result<(), RasterError> { + let file_len = reader.seek(SeekFrom::End(0))?; + reader.seek(SeekFrom::Start(0))?; + let checked = check_entries(reader, file_len); + reader.seek(SeekFrom::Start(0))?; + match checked { + // A short read means the tiff decoder refuses the same bytes. + Err(Short::Eof) | Ok(()) => Ok(()), + Err(Short::Refused(error)) => Err(error), + Err(Short::Io(error)) => Err(RasterError::from(error)), + } +} + +/// Why the entry walk stopped early. +enum Short { + Eof, + Io(io::Error), + Refused(RasterError), +} + +impl From for Short { + fn from(error: io::Error) -> Self { + if error.kind() == io::ErrorKind::UnexpectedEof { + Short::Eof + } else { + Short::Io(error) + } + } +} + +fn check_entries(reader: &mut R, file_len: u64) -> Result<(), Short> { + let mut header = [0u8; 8]; + reader.read_exact(&mut header)?; + let order = match &header[..2] { + b"II" => Order::Little, + b"MM" => Order::Big, + _ => return Ok(()), + }; + let magic = u16_at(order, [header[2], header[3]]); + let (format, ifd_offset) = match magic { + CLASSIC_MAGIC => { + let offset = u32_at(order, [header[4], header[5], header[6], header[7]]); + (Format { order, big: false }, u64::from(offset)) + } + BIG_MAGIC => { + let mut offset = [0u8; 8]; + reader.read_exact(&mut offset)?; + (Format { order, big: true }, u64_at(order, offset)) + } + _ => return Ok(()), + }; + if ifd_offset >= file_len { + return Ok(()); + } + reader.seek(SeekFrom::Start(ifd_offset))?; + let count = if format.big { + let mut raw = [0u8; 8]; + reader.read_exact(&mut raw)?; + u64_at(order, raw) + } else { + let mut raw = [0u8; 2]; + reader.read_exact(&mut raw)?; + u64::from(u16_at(order, raw)) + }; + // Entries past the end of the file are a short read for the decoder. + let room = file_len.saturating_sub(ifd_offset) / format.entry_len(); + for _ in 0..count.min(room) { + check_entry(reader, format, file_len)?; + } + Ok(()) +} + +fn check_entry(reader: &mut R, format: Format, file_len: u64) -> Result<(), Short> { + let order = format.order; + let mut head = [0u8; 4]; + reader.read_exact(&mut head)?; + let tag = u16_at(order, [head[0], head[1]]); + let kind = u16_at(order, [head[2], head[3]]); + let (count, offset) = if format.big { + let mut rest = [0u8; 16]; + reader.read_exact(&mut rest)?; + let mut count = [0u8; 8]; + let mut value = [0u8; 8]; + count.copy_from_slice(&rest[..8]); + value.copy_from_slice(&rest[8..]); + (u64_at(order, count), u64_at(order, value)) + } else { + let mut rest = [0u8; 8]; + reader.read_exact(&mut rest)?; + let count = u32_at(order, [rest[0], rest[1], rest[2], rest[3]]); + let value = u32_at(order, [rest[4], rest[5], rest[6], rest[7]]); + (u64::from(count), u64::from(value)) + }; + // The decoder skips an entry of unknown type, so this pass does too. + let Some(width) = type_width(kind) else { + return Ok(()); + }; + let refuse = |bytes| { + Short::Refused(RasterError::TagValueOutOfBounds { + tag, + bytes, + file_len, + }) + }; + let bytes = count.checked_mul(width).ok_or(refuse(u64::MAX))?; + if bytes <= format.inline_len() { + return Ok(()); + } + let end = offset.checked_add(bytes).ok_or(refuse(bytes))?; + if bytes > MAX_TAG_VALUE_BYTES as u64 || end > file_len { + return Err(refuse(bytes)); + } + Ok(()) +} + +/// Bytes per value of TIFF field type `kind`, or `None` for an unknown +/// type. +fn type_width(kind: u16) -> Option { + match kind { + // BYTE, ASCII, SBYTE, UNDEFINED. + 1 | 2 | 6 | 7 => Some(1), + // SHORT, SSHORT. + 3 | 8 => Some(2), + // LONG, SLONG, FLOAT, IFD. + 4 | 9 | 11 | 13 => Some(4), + // RATIONAL, SRATIONAL, DOUBLE, LONG8, SLONG8, IFD8. + 5 | 10 | 12 | 16 | 17 | 18 => Some(8), + _ => None, + } +} + +fn u16_at(order: Order, raw: [u8; 2]) -> u16 { + match order { + Order::Little => u16::from_le_bytes(raw), + Order::Big => u16::from_be_bytes(raw), + } +} + +fn u32_at(order: Order, raw: [u8; 4]) -> u32 { + match order { + Order::Little => u32::from_le_bytes(raw), + Order::Big => u32::from_be_bytes(raw), + } +} + +fn u64_at(order: Order, raw: [u8; 8]) -> u64 { + match order { + Order::Little => u64::from_le_bytes(raw), + Order::Big => u64::from_be_bytes(raw), + } +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + + use super::*; + + /// A little-endian classic TIFF: header, then one IFD of `entries`. + fn classic(entries: &[(u16, u16, u32, u32)], padding: usize) -> Vec { + let mut out = b"II".to_vec(); + out.extend(42u16.to_le_bytes()); + out.extend(8u32.to_le_bytes()); + out.extend(u16::try_from(entries.len()).expect("few").to_le_bytes()); + for (tag, kind, count, value) in entries { + out.extend(tag.to_le_bytes()); + out.extend(kind.to_le_bytes()); + out.extend(count.to_le_bytes()); + out.extend(value.to_le_bytes()); + } + out.extend(0u32.to_le_bytes()); + out.resize(out.len() + padding, 0); + out + } + + fn check(bytes: Vec) -> Result<(), RasterError> { + let mut cursor = Cursor::new(bytes); + let result = check_ifd0(&mut cursor); + assert_eq!(cursor.position(), 0, "the pass rewinds"); + result + } + + #[test] + fn values_inside_the_file_pass() { + // ImageWidth inline, StripOffsets: 4 LONGs at offset 26. + let bytes = classic(&[(256, 3, 1, 64), (273, 4, 4, 26)], 16); + assert!(check(bytes).is_ok()); + } + + #[test] + fn a_value_past_the_end_is_refused() { + let bytes = classic(&[(273, 4, 1_000_000, 26)], 16); + assert!(matches!( + check(bytes), + Err(RasterError::TagValueOutOfBounds { + tag: 273, + bytes: 4_000_000, + .. + }) + )); + } + + #[test] + fn a_value_over_the_cap_is_refused_even_inside_the_file() { + let count = u32::try_from(MAX_TAG_VALUE_BYTES + 1).expect("fits"); + let bytes = classic(&[(270, 2, count, 26)], MAX_TAG_VALUE_BYTES + 64); + assert!(matches!( + check(bytes), + Err(RasterError::TagValueOutOfBounds { tag: 270, .. }) + )); + } + + #[test] + fn unknown_types_and_short_tables_are_left_to_the_decoder() { + let unknown = classic(&[(273, 99, u32::MAX, 0)], 0); + assert!(check(unknown).is_ok()); + let mut short = classic(&[(273, 4, 4, 26)], 0); + short.truncate(12); + assert!(check(short).is_ok()); + assert!(check(b"not a tiff".to_vec()).is_ok()); + assert!(check(Vec::new()).is_ok()); + } + + #[test] + fn bigtiff_entries_are_checked() { + let mut out = b"II".to_vec(); + out.extend(43u16.to_le_bytes()); + out.extend(8u16.to_le_bytes()); + out.extend(0u16.to_le_bytes()); + out.extend(16u64.to_le_bytes()); + out.extend(1u64.to_le_bytes()); + out.extend(273u16.to_le_bytes()); + out.extend(16u16.to_le_bytes()); + out.extend(u64::MAX.to_le_bytes()); + out.extend(0u64.to_le_bytes()); + out.extend(0u64.to_le_bytes()); + assert!(matches!( + check(out), + Err(RasterError::TagValueOutOfBounds { tag: 273, .. }) + )); + } +} diff --git a/nodedb-raster/src/reader.rs b/nodedb-raster/src/reader.rs new file mode 100644 index 000000000..93ba1877c --- /dev/null +++ b/nodedb-raster/src/reader.rs @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! `GeoTiffReader`: IFD 0 of a GeoTIFF, decoded one block at a time. +//! +//! Open bounds every IFD 0 tag value by the file size first. It then reads +//! the tags once and refuses what the reader cannot decode exactly: an +//! overview as IFD 0, an unsupported codec, sample type, or photometric +//! interpretation, and a block larger than the caller's cap. A block read +//! then allocates at most that cap. + +use std::io::{Read, Seek}; +use std::mem::size_of; + +use nodedb_types::decode_bounds::checked_decode_capacity; +use nodedb_types::geometry::Srid; +use tiff::decoder::{Decoder, DecodingResult}; +use tiff::tags::Tag; + +use crate::block::{BandSamples, ChunkShape, RasterBlock, chunky_bands, planar_band}; +use crate::crs::srid_from_geokeys; +use crate::error::RasterError; +use crate::geokeys::{GT_RASTER_TYPE, GeoKeyDirectory}; +use crate::layout::{Interleave, RasterLayout}; +use crate::limits::RasterLimits; +use crate::nodata::parse_gdal_nodata; +use crate::preflight::check_ifd0; +use crate::sample::SampleType; +use crate::transform::{GeoTransform, ModelTags}; + +/// Compression codes the decoder reads: none, LZW, deflate (both codes), +/// PackBits. +const SUPPORTED_COMPRESSION: [u16; 5] = [1, 5, 8, 32946, 32773]; +/// `NewSubfileType` bit of a reduced-resolution image. +const REDUCED_RESOLUTION_BIT: u32 = 1; +/// `SubfileType` value of a reduced-resolution image. +const SUBFILE_REDUCED_RESOLUTION: u32 = 2; + +/// A GeoTIFF opened at IFD 0. +pub struct GeoTiffReader { + decoder: Decoder, + layout: RasterLayout, + limits: RasterLimits, + transform: Option, + geokeys: Option, + nodata: Option, +} + +impl GeoTiffReader { + /// Open `reader` and read the tags of IFD 0. + pub fn open(mut reader: R, limits: RasterLimits) -> Result { + let cap = limits.block_bytes; + let tiff = |e| RasterError::from_tiff(e, cap); + check_ifd0(&mut reader)?; + let mut decoder = Decoder::new(reader) + .map_err(tiff)? + .with_limits(limits.tiff_limits()); + refuse_reduced_resolution(&mut decoder, cap)?; + let compression = decoder + .find_tag_unsigned::(Tag::Compression) + .map_err(tiff)? + .unwrap_or(1); + if !SUPPORTED_COMPRESSION.contains(&compression) { + return Err(RasterError::UnsupportedCompression { code: compression }); + } + let layout = RasterLayout::read(&mut decoder, cap)?; + let needed_bytes = layout.block_bytes(); + if needed_bytes > cap as u64 { + return Err(RasterError::BlockTooLarge { + needed_bytes, + limit_bytes: cap, + }); + } + let geokeys = read_geokeys(&mut decoder, cap)?; + let pixel_scale = f64_tag(&mut decoder, Tag::ModelPixelScaleTag, cap)?; + let tiepoints = f64_tag(&mut decoder, Tag::ModelTiepointTag, cap)?; + let transformation = f64_tag(&mut decoder, Tag::ModelTransformationTag, cap)?; + let transform = GeoTransform::from_tags(ModelTags { + pixel_scale: pixel_scale.as_deref(), + tiepoints: tiepoints.as_deref(), + transformation: transformation.as_deref(), + raster_type: geokeys.as_ref().and_then(|keys| keys.short(GT_RASTER_TYPE)), + })?; + let nodata = decoder + .find_tag(Tag::GdalNodata) + .map_err(tiff)? + .map(|value| value.into_string().map_err(tiff)) + .transpose()? + .map(|text| parse_gdal_nodata(&text)) + .transpose()?; + Ok(Self { + decoder, + layout, + limits, + transform, + geokeys, + nodata, + }) + } + + /// The block grid and band layout. + pub fn layout(&self) -> &RasterLayout { + &self.layout + } + + /// Image width in pixels. + pub fn width(&self) -> u32 { + self.layout.width + } + + /// Image height in pixels. + pub fn height(&self) -> u32 { + self.layout.height + } + + /// Number of bands. + pub fn band_count(&self) -> u16 { + self.layout.bands + } + + /// The sample type every band shares. + pub fn sample_type(&self) -> SampleType { + self.layout.sample_type + } + + /// Blocks per band plane. Block indexes run `0..block_count()`. + pub fn block_count(&self) -> u32 { + self.layout.block_count() + } + + /// The pixel-corner geotransform, PixelIsPoint already shifted. + pub fn transform(&self) -> Option { + self.transform + } + + /// The GeoKey directory, when the file has one. + pub fn geokeys(&self) -> Option<&GeoKeyDirectory> { + self.geokeys.as_ref() + } + + /// The SRID the GeoKeys name, or `None` when they name no CRS. + pub fn srid(&self) -> Result, RasterError> { + srid_from_geokeys(self.geokeys.as_ref()) + } + + /// The `GDAL_NODATA` value, shared by every band. + pub fn nodata(&self) -> Option { + self.nodata + } + + /// Decode block `index`: one sample vector per band, cropped to the + /// image. + pub fn read_block(&mut self, index: u32) -> Result { + let window = self.layout.window(index)?; + let shape = ChunkShape { + index, + pixels: window.pixels(), + bands: usize::from(self.layout.bands), + limit_bytes: self.limits.block_bytes, + }; + let bands = match self.layout.interleave { + Interleave::Chunky => chunky_bands(self.read_chunk(index)?, shape)?, + Interleave::Planar => self.read_planes(index, shape)?, + }; + Ok(RasterBlock { + index, + window, + bands, + }) + } + + /// Read block `index` of every band plane. Plane `b` holds its blocks + /// at chunk indexes `b * block_count() ..`. + fn read_planes( + &mut self, + index: u32, + shape: ChunkShape, + ) -> Result, RasterError> { + let count = self.layout.block_count(); + let capacity = checked_decode_capacity( + shape.bands, + size_of::(), + shape.bands, + 1, + usize::from(u16::MAX), + shape.limit_bytes, + ) + .ok_or(RasterError::BlockTooLarge { + needed_bytes: self.layout.block_bytes(), + limit_bytes: shape.limit_bytes, + })?; + let mut bands = Vec::with_capacity(capacity); + for band in 0..u32::from(self.layout.bands) { + let chunk = band + .checked_mul(count) + .and_then(|start| start.checked_add(index)) + .ok_or(RasterError::BlockOutOfRange { index, count })?; + bands.push(planar_band(self.read_chunk(chunk)?, shape)?); + } + Ok(bands) + } + + fn read_chunk(&mut self, chunk: u32) -> Result { + let cap = self.limits.block_bytes; + self.decoder + .read_chunk(chunk) + .map_err(|e| RasterError::from_tiff(e, cap)) + } +} + +/// Refuse an IFD 0 flagged as an overview by `NewSubfileType` or the +/// older `SubfileType`. +fn refuse_reduced_resolution( + decoder: &mut Decoder, + cap: usize, +) -> Result<(), RasterError> { + let tiff = |e| RasterError::from_tiff(e, cap); + let new_subfile = decoder + .find_tag_unsigned::(Tag::NewSubfileType) + .map_err(tiff)? + .unwrap_or(0); + let subfile = decoder + .find_tag_unsigned::(Tag::SubfileType) + .map_err(tiff)? + .unwrap_or(0); + if new_subfile & REDUCED_RESOLUTION_BIT != 0 || subfile == SUBFILE_REDUCED_RESOLUTION { + return Err(RasterError::ReducedResolutionFirstImage); + } + Ok(()) +} + +/// The GeoKey directory with its parameter arrays, when the file has one. +fn read_geokeys( + decoder: &mut Decoder, + cap: usize, +) -> Result, RasterError> { + let tiff = |e| RasterError::from_tiff(e, cap); + let Some(directory) = decoder + .find_tag(Tag::GeoKeyDirectoryTag) + .map_err(tiff)? + .map(|value| value.into_u16_vec().map_err(tiff)) + .transpose()? + else { + return Ok(None); + }; + let doubles = f64_tag(decoder, Tag::GeoDoubleParamsTag, cap)?; + let ascii = decoder + .find_tag(Tag::GeoAsciiParamsTag) + .map_err(tiff)? + .map(|value| value.into_string().map_err(tiff)) + .transpose()?; + let keys = GeoKeyDirectory::parse(&directory, doubles.as_deref(), ascii.as_deref())?; + Ok(Some(keys)) +} + +/// The doubles of `tag`, when present. +fn f64_tag( + decoder: &mut Decoder, + tag: Tag, + cap: usize, +) -> Result>, RasterError> { + let tiff = |e| RasterError::from_tiff(e, cap); + decoder + .find_tag(tag) + .map_err(tiff)? + .map(|value| value.into_f64_vec().map_err(tiff)) + .transpose() +} diff --git a/nodedb-raster/src/sample.rs b/nodedb-raster/src/sample.rs new file mode 100644 index 000000000..5d82b4a68 --- /dev/null +++ b/nodedb-raster/src/sample.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The closed set of sample types the reader decodes. + +use std::fmt; + +use tiff::tags::SampleFormat; + +use crate::error::RasterError; + +/// A TIFF `SampleFormat` tag value, kept for error messages. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SampleFormatCode(pub u16); + +impl fmt::Display for SampleFormatCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + 1 => f.write_str("unsigned integer (1)"), + 2 => f.write_str("signed integer (2)"), + 3 => f.write_str("IEEE float (3)"), + 4 => f.write_str("undefined (4)"), + code => write!(f, "code {code}"), + } + } +} + +/// One band sample type. Every band of a raster shares it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum SampleType { + U8, + U16, + I16, + U32, + F32, + F64, +} + +impl SampleType { + /// The sample type for a TIFF sample format and bit width. Any other + /// pair fails with [`RasterError::UnsupportedSampleType`]. + pub fn from_tiff(format: SampleFormat, bits: u8) -> Result { + match (format, bits) { + (SampleFormat::Uint, 8) => Ok(SampleType::U8), + (SampleFormat::Uint, 16) => Ok(SampleType::U16), + (SampleFormat::Int, 16) => Ok(SampleType::I16), + (SampleFormat::Uint, 32) => Ok(SampleType::U32), + (SampleFormat::IEEEFP, 32) => Ok(SampleType::F32), + (SampleFormat::IEEEFP, 64) => Ok(SampleType::F64), + _ => Err(RasterError::UnsupportedSampleType { + format: SampleFormatCode(format.to_u16()), + bits, + }), + } + } + + /// Bytes per sample. + pub const fn byte_len(self) -> usize { + match self { + SampleType::U8 => 1, + SampleType::U16 | SampleType::I16 => 2, + SampleType::U32 | SampleType::F32 => 4, + SampleType::F64 => 8, + } + } + + /// The lower-case Rust name, such as `u16`. + pub const fn name(self) -> &'static str { + match self { + SampleType::U8 => "u8", + SampleType::U16 => "u16", + SampleType::I16 => "i16", + SampleType::U32 => "u32", + SampleType::F32 => "f32", + SampleType::F64 => "f64", + } + } +} + +impl fmt::Display for SampleType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.name()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn supported_pairs_map_to_their_type() { + assert_eq!( + SampleType::from_tiff(SampleFormat::Uint, 8).ok(), + Some(SampleType::U8) + ); + assert_eq!( + SampleType::from_tiff(SampleFormat::Int, 16).ok(), + Some(SampleType::I16) + ); + assert_eq!( + SampleType::from_tiff(SampleFormat::IEEEFP, 64).ok(), + Some(SampleType::F64) + ); + } + + #[test] + fn unsupported_pairs_are_typed_errors() { + for (format, bits) in [ + (SampleFormat::Int, 8), + (SampleFormat::Uint, 64), + (SampleFormat::Int, 32), + (SampleFormat::IEEEFP, 16), + (SampleFormat::Uint, 1), + (SampleFormat::Void, 8), + ] { + assert!(matches!( + SampleType::from_tiff(format, bits), + Err(RasterError::UnsupportedSampleType { .. }) + )); + } + } +} diff --git a/nodedb-raster/src/transform.rs b/nodedb-raster/src/transform.rs new file mode 100644 index 000000000..4e97b0790 --- /dev/null +++ b/nodedb-raster/src/transform.rs @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The pixel-to-world affine transform, from the GeoTIFF model tags. +//! +//! The transform uses GDAL order: `x = gt[0] + col·gt[1] + row·gt[2]` and +//! `y = gt[3] + col·gt[4] + row·gt[5]`, with `(col, row)` the pixel corner. +//! Tag precedence follows GDAL: a tiepoint with a pixel scale wins over +//! `ModelTransformationTag`. + +use crate::error::RasterError; + +/// `GTRasterTypeGeoKey` value of a raster whose tags name pixel centres. +const PIXEL_IS_POINT: u16 = 2; + +/// A GDAL-order affine geotransform from pixel corner to world coordinates. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct GeoTransform(pub [f64; 6]); + +/// The georeferencing tags of IFD 0, as read from the file. +#[derive(Debug, Clone, Copy, Default)] +pub struct ModelTags<'a> { + /// `ModelPixelScaleTag`: `(Sx, Sy, Sz)`. + pub pixel_scale: Option<&'a [f64]>, + /// `ModelTiepointTag`: `(I, J, K, X, Y, Z)` per tiepoint. + pub tiepoints: Option<&'a [f64]>, + /// `ModelTransformationTag`: a row-major 4x4 matrix. + pub transformation: Option<&'a [f64]>, + /// `GTRasterTypeGeoKey`, when the GeoKey directory holds it. + pub raster_type: Option, +} + +impl GeoTransform { + /// The transform the tags define, or `None` when the file carries no + /// model tags. + pub fn from_tags(tags: ModelTags<'_>) -> Result, RasterError> { + let base = match (tags.pixel_scale, tags.tiepoints, tags.transformation) { + (Some(scale), Some(tiepoints), _) => from_tiepoint(scale, tiepoints)?, + (Some(_), None, None) => return Err(RasterError::PixelScaleWithoutTiepoint), + (_, _, Some(matrix)) => from_matrix(matrix)?, + (None, Some(_), None) => return Err(RasterError::GroundControlPoints), + (None, None, None) => return Ok(None), + }; + let transform = if tags.raster_type == Some(PIXEL_IS_POINT) { + base.shifted_to_corner() + } else { + base + }; + transform.check()?; + Ok(Some(transform)) + } + + /// The world coordinate of the pixel corner `(col, row)`. + pub fn apply(&self, col: f64, row: f64) -> (f64, f64) { + let [x0, xc, xr, y0, yc, yr] = self.0; + (x0 + col * xc + row * xr, y0 + col * yc + row * yr) + } + + /// Move a PixelIsPoint origin from the first pixel's centre to its + /// corner, half a pixel up and left. + fn shifted_to_corner(self) -> Self { + let mut gt = self.0; + gt[0] -= 0.5 * gt[1] + 0.5 * gt[2]; + gt[3] -= 0.5 * gt[4] + 0.5 * gt[5]; + Self(gt) + } + + /// Refuse a non-finite term or a transform with a zero determinant. + fn check(&self) -> Result<(), RasterError> { + let gt = self.0; + let determinant = gt[1] * gt[5] - gt[2] * gt[4]; + if gt.iter().all(|term| term.is_finite()) && determinant != 0.0 && determinant.is_finite() { + Ok(()) + } else { + Err(RasterError::DegenerateTransform { transform: gt }) + } + } +} + +/// `GT = [X − I·Sx, Sx, 0, Y + J·Sy, 0, −Sy]` from the first tiepoint. +fn from_tiepoint(scale: &[f64], tiepoints: &[f64]) -> Result { + let [sx, sy, ..] = *scale else { + return Err(RasterError::GeoTagLength { + tag: "ModelPixelScaleTag", + found: scale.len(), + expected: "at least 2", + }); + }; + let [i, j, _k, x, y, ..] = *tiepoints else { + return Err(RasterError::GeoTagLength { + tag: "ModelTiepointTag", + found: tiepoints.len(), + expected: "at least 6", + }); + }; + if !tiepoints.len().is_multiple_of(6) { + return Err(RasterError::GeoTagLength { + tag: "ModelTiepointTag", + found: tiepoints.len(), + expected: "a multiple of 6", + }); + } + Ok(GeoTransform([x - i * sx, sx, 0.0, y + j * sy, 0.0, -sy])) +} + +/// `GT = [M3, M0, M1, M7, M4, M5]` from a 2D affine 4x4 matrix. +fn from_matrix(matrix: &[f64]) -> Result { + let Ok(m) = <[f64; 16]>::try_from(matrix) else { + return Err(RasterError::GeoTagLength { + tag: "ModelTransformationTag", + found: matrix.len(), + expected: "16", + }); + }; + let z_terms_zero = m[2] == 0.0 && m[6] == 0.0; + let last_row_identity = m[12] == 0.0 && m[13] == 0.0 && m[14] == 0.0 && m[15] == 1.0; + if !(z_terms_zero && last_row_identity) { + return Err(RasterError::NonAffineTransformation); + } + Ok(GeoTransform([m[3], m[0], m[1], m[7], m[4], m[5]])) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tags<'a>() -> ModelTags<'a> { + ModelTags::default() + } + + #[test] + fn no_model_tags_is_no_transform() { + assert_eq!(GeoTransform::from_tags(tags()).ok(), Some(None)); + } + + #[test] + fn tiepoint_and_scale() { + let scale = [30.0, 30.0, 0.0]; + let tie = [0.0, 0.0, 0.0, 600_000.0, 450_000.0, 0.0]; + let gt = GeoTransform::from_tags(ModelTags { + pixel_scale: Some(&scale), + tiepoints: Some(&tie), + ..tags() + }) + .expect("valid") + .expect("present"); + assert_eq!(gt.0, [600_000.0, 30.0, 0.0, 450_000.0, 0.0, -30.0]); + } + + #[test] + fn tiepoint_at_a_nonzero_pixel_moves_the_origin() { + let scale = [10.0, 5.0, 0.0]; + let tie = [2.0, 3.0, 0.0, 1000.0, 2000.0, 0.0]; + let gt = GeoTransform::from_tags(ModelTags { + pixel_scale: Some(&scale), + tiepoints: Some(&tie), + ..tags() + }) + .expect("valid") + .expect("present"); + assert_eq!(gt.0, [980.0, 10.0, 0.0, 2015.0, 0.0, -5.0]); + assert_eq!(gt.apply(2.0, 3.0), (1000.0, 2000.0)); + } + + #[test] + fn pixel_is_point_shifts_half_a_pixel() { + let scale = [10.0, 10.0, 0.0]; + let tie = [0.0, 0.0, 0.0, 100.0, 200.0, 0.0]; + let gt = GeoTransform::from_tags(ModelTags { + pixel_scale: Some(&scale), + tiepoints: Some(&tie), + raster_type: Some(PIXEL_IS_POINT), + ..tags() + }) + .expect("valid") + .expect("present"); + assert_eq!(gt.0, [95.0, 10.0, 0.0, 205.0, 0.0, -10.0]); + } + + #[test] + fn model_transformation_maps_to_gdal_order() { + let m = [ + 2.0, 0.5, 0.0, 100.0, // + 0.25, -3.0, 0.0, 200.0, // + 0.0, 0.0, 0.0, 0.0, // + 0.0, 0.0, 0.0, 1.0, + ]; + let gt = GeoTransform::from_tags(ModelTags { + transformation: Some(&m), + ..tags() + }) + .expect("valid") + .expect("present"); + assert_eq!(gt.0, [100.0, 2.0, 0.5, 200.0, 0.25, -3.0]); + } + + #[test] + fn rotated_pixel_is_point_shift_uses_both_terms() { + let m = [ + 2.0, 0.5, 0.0, 100.0, // + 0.25, -3.0, 0.0, 200.0, // + 0.0, 0.0, 0.0, 0.0, // + 0.0, 0.0, 0.0, 1.0, + ]; + let gt = GeoTransform::from_tags(ModelTags { + transformation: Some(&m), + raster_type: Some(PIXEL_IS_POINT), + ..tags() + }) + .expect("valid") + .expect("present"); + assert_eq!(gt.0, [98.75, 2.0, 0.5, 201.375, 0.25, -3.0]); + } + + #[test] + fn non_affine_matrix_is_refused() { + let mut m = [0.0; 16]; + m[0] = 1.0; + m[5] = -1.0; + m[2] = 4.0; + m[15] = 1.0; + assert!(matches!( + GeoTransform::from_tags(ModelTags { + transformation: Some(&m), + ..tags() + }), + Err(RasterError::NonAffineTransformation) + )); + m[2] = 0.0; + m[15] = 2.0; + assert!(matches!( + GeoTransform::from_tags(ModelTags { + transformation: Some(&m), + ..tags() + }), + Err(RasterError::NonAffineTransformation) + )); + } + + #[test] + fn tiepoints_without_scale_are_gcps() { + let tie = [0.0, 0.0, 0.0, 1.0, 2.0, 0.0, 10.0, 10.0, 0.0, 5.0, 6.0, 0.0]; + assert!(matches!( + GeoTransform::from_tags(ModelTags { + tiepoints: Some(&tie), + ..tags() + }), + Err(RasterError::GroundControlPoints) + )); + } + + #[test] + fn scale_without_tiepoint_is_refused() { + let scale = [1.0, 1.0, 0.0]; + assert!(matches!( + GeoTransform::from_tags(ModelTags { + pixel_scale: Some(&scale), + ..tags() + }), + Err(RasterError::PixelScaleWithoutTiepoint) + )); + } + + #[test] + fn short_or_degenerate_tags_are_refused() { + let scale = [1.0]; + let tie = [0.0; 6]; + assert!(matches!( + GeoTransform::from_tags(ModelTags { + pixel_scale: Some(&scale), + tiepoints: Some(&tie), + ..tags() + }), + Err(RasterError::GeoTagLength { .. }) + )); + let zero_scale = [0.0, 1.0, 0.0]; + assert!(matches!( + GeoTransform::from_tags(ModelTags { + pixel_scale: Some(&zero_scale), + tiepoints: Some(&tie), + ..tags() + }), + Err(RasterError::DegenerateTransform { .. }) + )); + let nan_scale = [f64::NAN, 1.0, 0.0]; + assert!(matches!( + GeoTransform::from_tags(ModelTags { + pixel_scale: Some(&nan_scale), + tiepoints: Some(&tie), + ..tags() + }), + Err(RasterError::DegenerateTransform { .. }) + )); + let short_matrix = [1.0; 9]; + assert!(matches!( + GeoTransform::from_tags(ModelTags { + transformation: Some(&short_matrix), + ..tags() + }), + Err(RasterError::GeoTagLength { .. }) + )); + } +} diff --git a/nodedb-raster/tests/geotiff/fixtures.rs b/nodedb-raster/tests/geotiff/fixtures.rs new file mode 100644 index 000000000..236d37d92 --- /dev/null +++ b/nodedb-raster/tests/geotiff/fixtures.rs @@ -0,0 +1,466 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! In-test GeoTIFF builders and whole-image readback. +//! +//! Strip fixtures go through the tiff `ImageEncoder`. Tiled, planar, and +//! floating-point-predictor fixtures write each chunk by hand through the +//! `DirectoryEncoder`, since `ImageEncoder` writes chunky strips only. + +use std::io::{Cursor, Read, Seek, Write}; + +use nodedb_raster::{GeoTiffReader, RasterError, RasterLimits}; +use tiff::encoder::colortype::ColorType; +use tiff::encoder::compression::{CompressionAlgorithm, Deflate, Lzw}; +use tiff::encoder::{Compression, DirectoryEncoder, Predictor, TiffEncoder, TiffKind, TiffValue}; +use tiff::tags::{ExtraSamples, Tag}; + +/// Decode cap for fixtures that are meant to fit. +pub const LIMITS: RasterLimits = RasterLimits::new(64 * 1024 * 1024); + +/// TIFF compression codes for hand-written chunks. +pub const NONE: u16 = 1; +pub const LZW: u16 = 5; +pub const DEFLATE: u16 = 8; + +/// One extra IFD 0 tag. +pub enum Extra { + Shorts(Tag, Vec), + Doubles(Tag, Vec), + Ascii(Tag, String), + Long(Tag, u32), +} + +fn write_extras( + dir: &mut DirectoryEncoder<'_, W, K>, + extras: &[Extra], +) { + for extra in extras { + let written = match extra { + Extra::Shorts(tag, values) => dir.write_tag(*tag, &values[..]), + Extra::Doubles(tag, values) => dir.write_tag(*tag, &values[..]), + Extra::Ascii(tag, text) => dir.write_tag(*tag, text.as_str()), + Extra::Long(tag, value) => dir.write_tag(*tag, *value), + }; + written.expect("extra tag writes"); + } +} + +/// A GeoKey directory of inline short keys, sorted ascending. +pub fn geokeys(entries: &[(u16, u16)]) -> Vec { + let count = u16::try_from(entries.len()).expect("few keys"); + let mut dir = vec![1, 1, 0, count]; + for (key, value) in entries { + dir.extend([*key, 0, 1, *value]); + } + dir +} + +/// A sample type the fixtures can write. +pub trait Sample: Copy + Default { + /// Append the native-endian bytes. + fn put(self, out: &mut Vec); + /// The big-endian bytes, for the floating-point predictor. + fn be(self) -> Vec; + /// A deterministic sample from a pattern counter. + fn from_pattern(counter: u32) -> Self; + fn to_f64(self) -> f64; + fn wrapping_diff(self, previous: Self) -> Self; +} + +macro_rules! int_sample { + ($t:ty, $offset:expr) => { + impl Sample for $t { + fn put(self, out: &mut Vec) { + out.extend_from_slice(&self.to_ne_bytes()); + } + fn be(self) -> Vec { + self.to_be_bytes().to_vec() + } + fn from_pattern(counter: u32) -> Self { + (counter as $t).wrapping_sub($offset) + } + fn to_f64(self) -> f64 { + f64::from(self) + } + fn wrapping_diff(self, previous: Self) -> Self { + self.wrapping_sub(previous) + } + } + }; +} + +macro_rules! float_sample { + ($t:ty) => { + impl Sample for $t { + fn put(self, out: &mut Vec) { + out.extend_from_slice(&self.to_ne_bytes()); + } + fn be(self) -> Vec { + self.to_be_bytes().to_vec() + } + fn from_pattern(counter: u32) -> Self { + (counter as $t) * 0.25 - 40.0 + } + fn to_f64(self) -> f64 { + f64::from(self) + } + fn wrapping_diff(self, previous: Self) -> Self { + self - previous + } + } + }; +} + +int_sample!(u8, 0); +int_sample!(u16, 0); +int_sample!(i16, 300); +int_sample!(u32, 0); +float_sample!(f32); +float_sample!(f64); + +/// One row-major plane per band, every sample distinct within a band. +pub fn planes(width: u32, height: u32, bands: u32) -> Vec> { + (0..bands) + .map(|band| { + (0..height) + .flat_map(|y| (0..width).map(move |x| band * 97 + y * 31 + x * 7 + 1)) + .map(T::from_pattern) + .collect() + }) + .collect() +} + +/// Interleave `planes` pixel by pixel, the chunky order. +pub fn interleave(planes: &[Vec]) -> Vec { + let pixels = planes.first().map_or(0, Vec::len); + (0..pixels) + .flat_map(|i| planes.iter().map(move |plane| plane[i])) + .collect() +} + +/// `planes` as `f64`, the shape `read_all` returns. +pub fn expected(planes: &[Vec]) -> Vec> { + planes + .iter() + .map(|plane| plane.iter().map(|s| s.to_f64()).collect()) + .collect() +} + +/// Options of a strip fixture written through `ImageEncoder`. +pub struct StripSpec<'a> { + pub width: u32, + pub height: u32, + pub compression: Compression, + pub predictor: Predictor, + pub rows_per_strip: Option, + pub extra_samples: usize, + pub extras: &'a [Extra], +} + +/// An uncompressed single-band strip spec. +pub fn strip_spec(width: u32, height: u32) -> StripSpec<'static> { + StripSpec { + width, + height, + compression: Compression::Uncompressed, + predictor: Predictor::None, + rows_per_strip: None, + extra_samples: 0, + extras: &[], + } +} + +/// Encode chunky strips of color type `C`. +pub fn encode_strips(spec: &StripSpec<'_>, data: &[C::Inner]) -> Vec +where + [C::Inner]: TiffValue, +{ + let mut cursor = Cursor::new(Vec::new()); + { + let mut encoder = TiffEncoder::new(&mut cursor) + .expect("encoder") + .with_compression(spec.compression) + .with_predictor(spec.predictor); + let mut image = encoder + .new_image::(spec.width, spec.height) + .expect("image"); + if spec.extra_samples > 0 { + image + .extra_samples(&vec![ExtraSamples::Unspecified; spec.extra_samples]) + .expect("extra samples"); + } + if let Some(rows) = spec.rows_per_strip { + image.rows_per_strip(rows).expect("rows per strip"); + } + write_extras(image.encoder(), spec.extras); + image.write_data(data).expect("image data writes"); + } + cursor.into_inner() +} + +/// How a hand-written image stores its chunks. +#[derive(Clone, Copy)] +pub enum RawLayout { + Strips { rows_per_strip: u32 }, + Tiles { width: u32, height: u32 }, +} + +/// A hand-written IFD 0. +pub struct RawImage<'a> { + pub width: u32, + pub height: u32, + pub bits: u16, + pub sample_format: u16, + pub samples: u16, + pub photometric: u16, + pub planar: bool, + pub compression: u16, + pub predictor: u16, + pub layout: RawLayout, + /// Encoded chunks in file order: every block of band 0 first when + /// planar. + pub chunks: Vec>, + pub extras: &'a [Extra], +} + +/// Write `image` as a standard TIFF. +pub fn encode_raw(image: &RawImage<'_>) -> Vec { + let mut cursor = Cursor::new(Vec::new()); + { + let mut encoder = TiffEncoder::new(&mut cursor).expect("encoder"); + let mut dir = encoder.image_directory().expect("directory"); + let mut offsets = Vec::new(); + let mut counts = Vec::new(); + for chunk in &image.chunks { + let offset = dir.write_data(&chunk[..]).expect("chunk writes"); + offsets.push(u32::try_from(offset).expect("small file")); + counts.push(u32::try_from(chunk.len()).expect("small chunk")); + } + let samples = usize::from(image.samples); + dir.write_tag(Tag::ImageWidth, image.width).expect("tag"); + dir.write_tag(Tag::ImageLength, image.height).expect("tag"); + dir.write_tag(Tag::BitsPerSample, &vec![image.bits; samples][..]) + .expect("tag"); + dir.write_tag(Tag::SampleFormat, &vec![image.sample_format; samples][..]) + .expect("tag"); + dir.write_tag(Tag::SamplesPerPixel, image.samples) + .expect("tag"); + dir.write_tag(Tag::PhotometricInterpretation, image.photometric) + .expect("tag"); + dir.write_tag( + Tag::PlanarConfiguration, + if image.planar { 2u16 } else { 1 }, + ) + .expect("tag"); + dir.write_tag(Tag::Compression, image.compression) + .expect("tag"); + dir.write_tag(Tag::Predictor, image.predictor).expect("tag"); + match image.layout { + RawLayout::Strips { rows_per_strip } => { + dir.write_tag(Tag::RowsPerStrip, rows_per_strip) + .expect("tag"); + dir.write_tag(Tag::StripOffsets, &offsets[..]).expect("tag"); + dir.write_tag(Tag::StripByteCounts, &counts[..]) + .expect("tag"); + } + RawLayout::Tiles { width, height } => { + dir.write_tag(Tag::TileWidth, width).expect("tag"); + dir.write_tag(Tag::TileLength, height).expect("tag"); + dir.write_tag(Tag::TileOffsets, &offsets[..]).expect("tag"); + dir.write_tag(Tag::TileByteCounts, &counts[..]) + .expect("tag"); + } + } + write_extras(&mut dir, image.extras); + dir.finish().expect("directory writes"); + } + cursor.into_inner() +} + +/// The predictor a hand-written chunk is encoded with. +#[derive(Clone, Copy, PartialEq)] +pub enum Pred { + None, + Horizontal, + FloatingPoint, +} + +impl Pred { + pub fn code(self) -> u16 { + match self { + Pred::None => 1, + Pred::Horizontal => 2, + Pred::FloatingPoint => 3, + } + } +} + +/// Split `planes` into chunk sample runs in file order. Tiles are padded +/// to full size. The last strip keeps only the image's rows. +pub fn chunk_samples( + planes: &[Vec], + width: u32, + height: u32, + layout: RawLayout, + planar: bool, +) -> Vec> { + let (chunk_w, chunk_h, crop_rows) = match layout { + RawLayout::Strips { rows_per_strip } => (width, rows_per_strip, true), + RawLayout::Tiles { width, height } => (width, height, false), + }; + let bands = planes.len(); + let groups: Vec> = if planar { + (0..bands).map(|band| vec![band]).collect() + } else { + vec![(0..bands).collect()] + }; + let mut chunks = Vec::new(); + for group in groups { + for ty in 0..height.div_ceil(chunk_h) { + for tx in 0..width.div_ceil(chunk_w) { + let mut chunk = Vec::new(); + for r in 0..chunk_h { + let y = ty * chunk_h + r; + if crop_rows && y >= height { + break; + } + for c in 0..chunk_w { + let x = tx * chunk_w + c; + for &band in &group { + chunk.push(if x < width && y < height { + planes[band][(y * width + x) as usize] + } else { + T::default() + }); + } + } + } + chunks.push(chunk); + } + } + } + chunks +} + +/// Encode one chunk: predictor per row, then compression. +pub fn chunk_bytes( + chunk: &[T], + row_samples: usize, + samples_per_pixel: usize, + predictor: Pred, + compression: u16, +) -> Vec { + let mut raw = Vec::new(); + for row in chunk.chunks(row_samples) { + match predictor { + Pred::None => row.iter().for_each(|s| s.put(&mut raw)), + Pred::Horizontal => { + for (i, sample) in row.iter().enumerate() { + let encoded = if i < samples_per_pixel { + *sample + } else { + sample.wrapping_diff(row[i - samples_per_pixel]) + }; + encoded.put(&mut raw); + } + } + Pred::FloatingPoint => raw.extend(float_predict(row, samples_per_pixel)), + } + } + compress(&raw, compression) +} + +/// Byte-plane shuffle of big-endian samples, then byte differencing at +/// the pixel stride (TIFF Technical Note 3). +fn float_predict(row: &[T], samples_per_pixel: usize) -> Vec { + let be: Vec> = row.iter().map(|s| s.be()).collect(); + let width = be.first().map_or(0, Vec::len); + let n = row.len(); + let mut shuffled = vec![0u8; n * width]; + for (i, bytes) in be.iter().enumerate() { + for (k, byte) in bytes.iter().enumerate() { + shuffled[k * n + i] = *byte; + } + } + for i in (samples_per_pixel..shuffled.len()).rev() { + shuffled[i] = shuffled[i].wrapping_sub(shuffled[i - samples_per_pixel]); + } + shuffled +} + +/// Compress `raw` with TIFF compression `code`. +pub fn compress(raw: &[u8], code: u16) -> Vec { + let mut out = Vec::new(); + match code { + NONE => out.extend_from_slice(raw), + LZW => { + Lzw.write_to(&mut out, raw).expect("lzw"); + } + DEFLATE => { + Deflate::default().write_to(&mut out, raw).expect("deflate"); + } + other => panic!("fixture compression {other} is not built"), + } + out +} + +/// Encoded chunks of `planes` under `layout`. +pub fn encode_chunks( + planes: &[Vec], + image: (u32, u32), + layout: RawLayout, + planar: bool, + predictor: Pred, + compression: u16, +) -> Vec> { + let chunk_w = match layout { + RawLayout::Strips { .. } => image.0, + RawLayout::Tiles { width: tile_w, .. } => tile_w, + }; + let spp = if planar { 1 } else { planes.len() }; + chunk_samples(planes, image.0, image.1, layout, planar) + .iter() + .map(|chunk| chunk_bytes(chunk, chunk_w as usize * spp, spp, predictor, compression)) + .collect() +} + +/// Open an owned copy of `bytes` under [`LIMITS`], so callers can pass a temporary. +pub fn open(bytes: &[u8]) -> Result>>, RasterError> { + GeoTiffReader::open(Cursor::new(bytes.to_vec()), LIMITS) +} + +/// Decode every block and reassemble one row-major `f64` plane per band. +pub fn read_all(reader: &mut GeoTiffReader) -> Vec> { + let width = reader.width() as usize; + let pixels = width * reader.height() as usize; + let mut out = vec![vec![f64::NAN; pixels]; usize::from(reader.band_count())]; + for index in 0..reader.block_count() { + let block = reader.read_block(index).expect("block decodes"); + let window = block.window; + assert_eq!(block.bands.len(), out.len(), "band count of block {index}"); + for (band, samples) in block.bands.iter().enumerate() { + assert_eq!(samples.len(), window.pixels(), "samples in block {index}"); + assert_eq!(samples.sample_type(), reader.sample_type()); + for r in 0..window.height as usize { + for c in 0..window.width as usize { + let row = window.row_off as usize + r; + let col = window.col_off as usize + c; + out[band][row * width + col] = samples + .value_f64(r * window.width as usize + c) + .expect("sample in window"); + } + } + } + } + out +} + +/// Open `bytes` with `limits` and decode every block; the first error. +pub fn decode_everything(bytes: &[u8], limits: RasterLimits) -> Result<(), RasterError> { + let mut reader = GeoTiffReader::open(Cursor::new(bytes), limits)?; + reader.srid()?; + for index in 0..reader.block_count() { + reader.read_block(index)?; + } + Ok(()) +} diff --git a/nodedb-raster/tests/geotiff/georef.rs b/nodedb-raster/tests/geotiff/georef.rs new file mode 100644 index 000000000..e7af7e33c --- /dev/null +++ b/nodedb-raster/tests/geotiff/georef.rs @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Georeference from the GeoTIFF tags: geotransform, CRS, and nodata. + +use nodedb_raster::RasterError; +use nodedb_raster::geokeys::{GEOGRAPHIC_TYPE, GT_MODEL_TYPE, GT_RASTER_TYPE, PROJECTED_CS_TYPE}; +use nodedb_types::geometry::Srid; +use tiff::encoder::colortype::Gray16; +use tiff::tags::Tag; + +use crate::fixtures::{Extra, StripSpec, encode_strips, geokeys, open, planes, strip_spec}; + +const W: u32 = 6; +const H: u32 = 4; + +fn with_tags(extras: &[Extra]) -> Vec { + let data = planes::(W, H, 1); + let spec = StripSpec { + extras, + ..strip_spec(W, H) + }; + encode_strips::(&spec, &data[0]) +} + +fn utm_tiepoint(raster_type: u16) -> Vec { + vec![ + Extra::Doubles(Tag::ModelPixelScaleTag, vec![30.0, 30.0, 0.0]), + Extra::Doubles( + Tag::ModelTiepointTag, + vec![0.0, 0.0, 0.0, 600_000.0, 450_000.0, 0.0], + ), + Extra::Shorts( + Tag::GeoKeyDirectoryTag, + geokeys(&[ + (GT_MODEL_TYPE, 1), + (GT_RASTER_TYPE, raster_type), + (PROJECTED_CS_TYPE, 32647), + ]), + ), + ] +} + +#[test] +fn tiepoint_and_scale_give_a_north_up_transform() { + let reader = open(&with_tags(&utm_tiepoint(1))).expect("opens"); + let gt = reader.transform().expect("georeferenced"); + assert_eq!(gt.0, [600_000.0, 30.0, 0.0, 450_000.0, 0.0, -30.0]); + assert_eq!(reader.srid().expect("CRS maps"), Some(Srid::Utm47N)); +} + +#[test] +fn pixel_is_point_shifts_the_origin_half_a_pixel() { + let reader = open(&with_tags(&utm_tiepoint(2))).expect("opens"); + let gt = reader.transform().expect("georeferenced"); + assert_eq!(gt.0, [599_985.0, 30.0, 0.0, 450_015.0, 0.0, -30.0]); +} + +#[test] +fn model_transformation_gives_a_rotated_transform() { + let matrix = vec![ + 0.5, 0.1, 0.0, 101.0, // + -0.1, -0.5, 0.0, 3.5, // + 0.0, 0.0, 0.0, 0.0, // + 0.0, 0.0, 0.0, 1.0, + ]; + let extras = [ + Extra::Doubles(Tag::ModelTransformationTag, matrix), + Extra::Shorts( + Tag::GeoKeyDirectoryTag, + geokeys(&[(GT_MODEL_TYPE, 2), (GEOGRAPHIC_TYPE, 4326)]), + ), + ]; + let reader = open(&with_tags(&extras)).expect("opens"); + let gt = reader.transform().expect("georeferenced"); + assert_eq!(gt.0, [101.0, 0.5, 0.1, 3.5, -0.1, -0.5]); + assert_eq!(reader.srid().expect("CRS maps"), Some(Srid::Wgs84)); +} + +#[test] +fn gdal_nodata_is_read() { + let reader = open(&with_tags(&[Extra::Ascii( + Tag::GdalNodata, + "-9999".to_owned(), + )])) + .expect("opens"); + assert_eq!(reader.nodata(), Some(-9999.0)); + + let reader = open(&with_tags(&[Extra::Ascii( + Tag::GdalNodata, + "nan ".to_owned(), + )])) + .expect("opens"); + assert!(reader.nodata().is_some_and(f64::is_nan)); + + let malformed = open(&with_tags(&[Extra::Ascii( + Tag::GdalNodata, + "none".to_owned(), + )])); + assert!(matches!(malformed, Err(RasterError::InvalidNodata { .. }))); +} + +#[test] +fn a_plain_tiff_has_no_georeference() { + let reader = open(&with_tags(&[])).expect("opens"); + assert_eq!(reader.transform(), None); + assert_eq!(reader.srid().expect("no CRS"), None); + assert_eq!(reader.nodata(), None); + assert!(reader.geokeys().is_none()); +} + +#[test] +fn ground_control_points_are_refused() { + let extras = [Extra::Doubles( + Tag::ModelTiepointTag, + vec![ + 0.0, 0.0, 0.0, 100.0, 200.0, 0.0, // + 5.0, 3.0, 0.0, 150.0, 170.0, 0.0, + ], + )]; + assert!(matches!( + open(&with_tags(&extras)), + Err(RasterError::GroundControlPoints) + )); +} + +#[test] +fn an_unsupported_crs_is_a_typed_error() { + let extras = [Extra::Shorts( + Tag::GeoKeyDirectoryTag, + geokeys(&[(GT_MODEL_TYPE, 1), (PROJECTED_CS_TYPE, 32633)]), + )]; + let reader = open(&with_tags(&extras)).expect("opens"); + assert!(matches!(reader.srid(), Err(RasterError::UnsupportedCrs(_)))); +} + +#[test] +fn a_malformed_geokey_directory_fails_open() { + let extras = [Extra::Shorts( + Tag::GeoKeyDirectoryTag, + vec![1, 1, 0, 9, 1024], + )]; + assert!(matches!( + open(&with_tags(&extras)), + Err(RasterError::GeoKeys(_)) + )); +} diff --git a/nodedb-raster/tests/geotiff/main.rs b/nodedb-raster/tests/geotiff/main.rs new file mode 100644 index 000000000..84ead462c --- /dev/null +++ b/nodedb-raster/tests/geotiff/main.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! `GeoTiffReader` over GeoTIFFs the tiff encoder builds in-test. + +mod fixtures; +mod georef; +mod robustness; +mod strips; +mod tiles; diff --git a/nodedb-raster/tests/geotiff/robustness.rs b/nodedb-raster/tests/geotiff/robustness.rs new file mode 100644 index 000000000..c22e35ba4 --- /dev/null +++ b/nodedb-raster/tests/geotiff/robustness.rs @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Refusals and hostile input: every failure is a typed error, never a +//! panic and never an allocation past the cap. + +use nodedb_raster::geokeys::{GT_MODEL_TYPE, PROJECTED_CS_TYPE}; +use nodedb_raster::{RasterError, RasterLimits}; +use tiff::encoder::colortype::{Gray16, Gray64, GrayI8}; +use tiff::encoder::{Compression, DeflateLevel}; +use tiff::tags::Tag; + +use crate::fixtures::{ + DEFLATE, Extra, LIMITS, Pred, RawImage, RawLayout, StripSpec, decode_everything, encode_chunks, + encode_raw, encode_strips, geokeys, open, planes, strip_spec, +}; + +/// A small planar tiled GeoTIFF with GeoKeys: every code path the reader +/// has, in a few hundred bytes. +fn small_fixture() -> Vec { + let (width, height) = (12, 10); + let layout = RawLayout::Tiles { + width: 16, + height: 16, + }; + let bands = planes::(width, height, 2); + let chunks = encode_chunks( + &bands, + (width, height), + layout, + true, + Pred::Horizontal, + DEFLATE, + ); + let extras = [ + Extra::Doubles(Tag::ModelPixelScaleTag, vec![10.0, 10.0, 0.0]), + Extra::Doubles( + Tag::ModelTiepointTag, + vec![0.0, 0.0, 0.0, 500_000.0, 300_000.0, 0.0], + ), + Extra::Shorts( + Tag::GeoKeyDirectoryTag, + geokeys(&[(GT_MODEL_TYPE, 1), (PROJECTED_CS_TYPE, 32648)]), + ), + Extra::Ascii(Tag::GdalNodata, "0".to_owned()), + ]; + encode_raw(&RawImage { + width, + height, + bits: 16, + sample_format: 1, + samples: 2, + photometric: 1, + planar: true, + compression: DEFLATE, + predictor: Pred::Horizontal.code(), + layout, + chunks, + extras: &extras, + }) +} + +#[test] +fn the_small_fixture_decodes() { + decode_everything(&small_fixture(), LIMITS).expect("fixture is valid"); +} + +#[test] +fn an_oversize_strip_is_a_typed_limit_error() { + let data = planes::(64, 64, 1); + let spec = StripSpec { + rows_per_strip: Some(64), + ..strip_spec(64, 64) + }; + let bytes = encode_strips::(&spec, &data[0]); + let tight = RasterLimits::new(4096); + assert!(matches!( + decode_everything(&bytes, tight), + Err(RasterError::BlockTooLarge { + needed_bytes: 8192, + limit_bytes: 4096 + }) + )); + decode_everything(&bytes, RasterLimits::new(8192)).expect("fits at the cap"); +} + +#[test] +fn an_oversize_tile_is_a_typed_limit_error() { + let tight = RasterLimits::new(16 * 16 * 2 * 2 - 1); + assert!(matches!( + decode_everything(&small_fixture(), tight), + Err(RasterError::BlockTooLarge { .. }) + )); +} + +#[test] +fn every_truncation_is_a_typed_error() { + let bytes = small_fixture(); + for len in 0..bytes.len() { + let result = decode_everything(&bytes[..len], LIMITS); + assert!(result.is_err(), "a {len}-byte prefix decoded"); + } +} + +#[test] +fn corrupt_bytes_never_panic() { + let bytes = small_fixture(); + for position in 0..bytes.len() { + for flip in [0xFF_u8, 0x01, 0x80] { + let mut corrupt = bytes.clone(); + corrupt[position] ^= flip; + // Ok or a typed error are both fine. A panic fails the test. + let _ = decode_everything(&corrupt, RasterLimits::new(1 << 20)); + } + } +} + +#[test] +fn garbage_is_malformed() { + for bytes in [ + &b""[..], + b"II", + b"hello world", + b"MM\x00\x2a\xff\xff\xff\xff", + ] { + assert!( + matches!(open(bytes), Err(RasterError::Malformed(_))), + "{bytes:?}" + ); + } +} + +#[test] +fn a_reduced_resolution_first_ifd_is_refused() { + let data = planes::(4, 4, 1); + let extras = [Extra::Long(Tag::NewSubfileType, 1)]; + let spec = StripSpec { + extras: &extras, + ..strip_spec(4, 4) + }; + assert!(matches!( + open(&encode_strips::(&spec, &data[0])), + Err(RasterError::ReducedResolutionFirstImage) + )); +} + +#[test] +fn unsupported_sample_types_are_refused_at_open() { + let spec = StripSpec { + compression: Compression::Deflate(DeflateLevel::Fast), + ..strip_spec(4, 4) + }; + let signed_bytes = vec![-3_i8; 16]; + assert!(matches!( + open(&encode_strips::(&spec, &signed_bytes)), + Err(RasterError::UnsupportedSampleType { bits: 8, .. }) + )); + let wide = vec![7_u64; 16]; + assert!(matches!( + open(&encode_strips::(&spec, &wide)), + Err(RasterError::UnsupportedSampleType { bits: 64, .. }) + )); +} + +fn raw_gray16(compression: u16, photometric: u16) -> Vec { + let layout = RawLayout::Strips { rows_per_strip: 4 }; + let bands = planes::(4, 4, 1); + encode_raw(&RawImage { + width: 4, + height: 4, + bits: 16, + sample_format: 1, + samples: 1, + photometric, + planar: false, + compression, + predictor: 1, + layout, + chunks: encode_chunks(&bands, (4, 4), layout, false, Pred::None, DEFLATE), + extras: &[], + }) +} + +#[test] +fn unsupported_compression_is_refused_at_open() { + // 50000 is ZSTD, 34887 LERC, 7 JPEG. + for code in [50000, 34887, 7] { + assert!(matches!( + open(&raw_gray16(code, 1)), + Err(RasterError::UnsupportedCompression { code: c }) if c == code + )); + } +} + +#[test] +fn transforming_photometrics_are_refused_at_open() { + // The decoder inverts WhiteIsZero samples. A palette stores indexes. + for photometric in [0, 3] { + assert!(matches!( + open(&raw_gray16(DEFLATE, photometric)), + Err(RasterError::UnsupportedPhotometric { code }) if code == photometric + )); + } +} + +#[test] +fn an_out_of_range_block_is_a_typed_error() { + let mut reader = open(&small_fixture()).expect("opens"); + let count = reader.block_count(); + assert!(matches!( + reader.read_block(count), + Err(RasterError::BlockOutOfRange { .. }) + )); +} diff --git a/nodedb-raster/tests/geotiff/strips.rs b/nodedb-raster/tests/geotiff/strips.rs new file mode 100644 index 000000000..43dadb965 --- /dev/null +++ b/nodedb-raster/tests/geotiff/strips.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Strip GeoTIFFs: every codec, both integer and float predictors, every +//! supported sample type, and multi-band chunky pixels. + +use nodedb_raster::{BlockKind, Interleave, SampleType}; +use tiff::encoder::colortype::{ + ColorType, Gray8, Gray16, Gray32, Gray32Float, Gray64Float, GrayI16, RGB8, RGB16, +}; +use tiff::encoder::{Compression, DeflateLevel, Predictor, TiffValue}; + +use crate::fixtures::{ + DEFLATE, NONE, Pred, RawImage, RawLayout, Sample, StripSpec, encode_chunks, encode_raw, + encode_strips, expected, interleave, open, planes, read_all, strip_spec, +}; + +const W: u32 = 10; +const H: u32 = 7; +const DEFLATE_BALANCED: Compression = Compression::Deflate(DeflateLevel::Balanced); + +#[test] +fn every_strip_codec_round_trips() { + let planes = planes::(W, H, 1); + // PackBits keeps the encoder's one row per strip. + let cases = [ + (Compression::Uncompressed, Some(3), 3), + (Compression::Lzw, Some(3), 3), + (DEFLATE_BALANCED, Some(3), 3), + (Compression::Packbits, None, H), + ]; + for (compression, rows_per_strip, blocks) in cases { + let spec = StripSpec { + compression, + rows_per_strip, + ..strip_spec(W, H) + }; + let bytes = encode_strips::(&spec, &planes[0]); + let mut reader = open(&bytes).expect("opens"); + assert_eq!(reader.layout().kind, BlockKind::Strips); + assert_eq!(reader.block_count(), blocks); + assert_eq!(read_all(&mut reader), expected(&planes)); + } +} + +#[test] +fn last_strip_is_cropped_to_the_image() { + let planes = planes::(W, H, 1); + let spec = StripSpec { + rows_per_strip: Some(3), + ..strip_spec(W, H) + }; + let bytes = encode_strips::(&spec, &planes[0]); + let mut reader = open(&bytes).expect("opens"); + let last = reader.read_block(2).expect("last strip"); + assert_eq!( + (last.window.row_off, last.window.width, last.window.height), + (6, W, 1) + ); + assert_eq!(last.bands[0].len(), W as usize); +} + +#[test] +fn horizontal_predictor_round_trips() { + let gray = planes::(W, H, 1); + let spec = StripSpec { + compression: Compression::Lzw, + predictor: Predictor::Horizontal, + rows_per_strip: Some(2), + ..strip_spec(W, H) + }; + let mut reader = open(&encode_strips::(&spec, &gray[0])).expect("opens"); + assert_eq!(read_all(&mut reader), expected(&gray)); + + let signed = planes::(W, H, 1); + let spec = StripSpec { + compression: DEFLATE_BALANCED, + predictor: Predictor::Horizontal, + ..strip_spec(W, H) + }; + let mut reader = open(&encode_strips::(&spec, &signed[0])).expect("opens"); + assert_eq!(read_all(&mut reader), expected(&signed)); + + let rgb = planes::(W, H, 3); + let spec = StripSpec { + compression: DEFLATE_BALANCED, + predictor: Predictor::Horizontal, + rows_per_strip: Some(4), + ..strip_spec(W, H) + }; + let mut reader = open(&encode_strips::(&spec, &interleave(&rgb))).expect("opens"); + assert_eq!(reader.band_count(), 3); + assert_eq!(read_all(&mut reader), expected(&rgb)); +} + +#[test] +fn floating_point_predictor_round_trips() { + let layout = RawLayout::Strips { rows_per_strip: 3 }; + for compression in [NONE, DEFLATE] { + let two_band = planes::(W, H, 2); + let chunks = encode_chunks( + &two_band, + (W, H), + layout, + false, + Pred::FloatingPoint, + compression, + ); + let bytes = encode_raw(&RawImage { + width: W, + height: H, + bits: 32, + sample_format: 3, + samples: 2, + photometric: 1, + planar: false, + compression, + predictor: Pred::FloatingPoint.code(), + layout, + chunks, + extras: &[], + }); + let mut reader = open(&bytes).expect("opens"); + assert_eq!(reader.sample_type(), SampleType::F32); + assert_eq!(read_all(&mut reader), expected(&two_band)); + } + + let single = planes::(W, H, 1); + let chunks = encode_chunks(&single, (W, H), layout, false, Pred::FloatingPoint, DEFLATE); + let bytes = encode_raw(&RawImage { + width: W, + height: H, + bits: 64, + sample_format: 3, + samples: 1, + photometric: 1, + planar: false, + compression: DEFLATE, + predictor: Pred::FloatingPoint.code(), + layout, + chunks, + extras: &[], + }); + let mut reader = open(&bytes).expect("opens"); + assert_eq!(reader.sample_type(), SampleType::F64); + assert_eq!(read_all(&mut reader), expected(&single)); +} + +fn round_trip(sample_type: SampleType) +where + C::Inner: Sample, + [C::Inner]: TiffValue, +{ + let planes = planes::(W, H, 1); + let spec = StripSpec { + compression: DEFLATE_BALANCED, + rows_per_strip: Some(2), + ..strip_spec(W, H) + }; + let mut reader = open(&encode_strips::(&spec, &planes[0])).expect("opens"); + assert_eq!(reader.sample_type(), sample_type); + assert_eq!(read_all(&mut reader), expected(&planes)); +} + +#[test] +fn every_supported_sample_type_round_trips() { + round_trip::(SampleType::U8); + round_trip::(SampleType::U16); + round_trip::(SampleType::I16); + round_trip::(SampleType::U32); + round_trip::(SampleType::F32); + round_trip::(SampleType::F64); +} + +#[test] +fn multi_band_chunky_pixels_split_per_band() { + let multiband = planes::(W, H, 4); + let spec = StripSpec { + compression: Compression::Lzw, + rows_per_strip: Some(4), + extra_samples: 3, + ..strip_spec(W, H) + }; + let mut reader = open(&encode_strips::(&spec, &interleave(&multiband))).expect("opens"); + assert_eq!(reader.band_count(), 4); + assert_eq!(reader.layout().interleave, Interleave::Chunky); + assert_eq!(read_all(&mut reader), expected(&multiband)); + + let rgb = planes::(W, H, 3); + let spec = StripSpec { + compression: Compression::Packbits, + ..strip_spec(W, H) + }; + let mut reader = open(&encode_strips::(&spec, &interleave(&rgb))).expect("opens"); + assert_eq!(read_all(&mut reader), expected(&rgb)); +} diff --git a/nodedb-raster/tests/geotiff/tiles.rs b/nodedb-raster/tests/geotiff/tiles.rs new file mode 100644 index 000000000..b143316a8 --- /dev/null +++ b/nodedb-raster/tests/geotiff/tiles.rs @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Tiled and planar GeoTIFFs, with edge tiles that overhang the image. + +use nodedb_raster::{BlockKind, BlockWindow, Interleave, SampleType}; + +use crate::fixtures::{ + DEFLATE, LZW, NONE, Pred, RawImage, RawLayout, Sample, encode_chunks, encode_raw, expected, + open, planes, read_all, +}; + +/// 40 x 24 pixels in 16 x 16 tiles: the right column is 8 wide and the +/// bottom row 8 tall. +const W: u32 = 40; +const H: u32 = 24; +const TILE: RawLayout = RawLayout::Tiles { + width: 16, + height: 16, +}; + +struct Case { + bands: u32, + planar: bool, + layout: RawLayout, + predictor: Pred, + compression: u16, +} + +fn encode(case: &Case, bits: u16, sample_format: u16) -> (Vec, Vec>) { + let bands = planes::(W, H, case.bands); + let chunks = encode_chunks( + &bands, + (W, H), + case.layout, + case.planar, + case.predictor, + case.compression, + ); + let bytes = encode_raw(&RawImage { + width: W, + height: H, + bits, + sample_format, + samples: u16::try_from(case.bands).expect("few bands"), + photometric: 1, + planar: case.planar, + compression: case.compression, + predictor: case.predictor.code(), + layout: case.layout, + chunks, + extras: &[], + }); + (bytes, expected(&bands)) +} + +#[test] +fn chunky_tiles_crop_edge_tiles() { + let case = Case { + bands: 3, + planar: false, + layout: TILE, + predictor: Pred::None, + compression: DEFLATE, + }; + let (bytes, want) = encode::(&case, 16, 1); + let mut reader = open(&bytes).expect("opens"); + assert_eq!(reader.layout().kind, BlockKind::Tiles); + assert_eq!(reader.layout().interleave, Interleave::Chunky); + assert_eq!(reader.block_count(), 6); + let corner = reader.read_block(5).expect("corner tile"); + assert_eq!( + corner.window, + BlockWindow { + col_off: 32, + row_off: 16, + width: 8, + height: 8 + } + ); + assert!(corner.bands.iter().all(|band| band.len() == 64)); + assert_eq!(read_all(&mut reader), want); +} + +#[test] +fn planar_tiles_crop_edge_tiles_in_every_plane() { + let case = Case { + bands: 2, + planar: true, + layout: TILE, + predictor: Pred::None, + compression: LZW, + }; + let (bytes, want) = encode::(&case, 16, 2); + let mut reader = open(&bytes).expect("opens"); + assert_eq!(reader.layout().interleave, Interleave::Planar); + assert_eq!(reader.sample_type(), SampleType::I16); + let bottom = reader.read_block(4).expect("bottom tile"); + assert_eq!((bottom.window.width, bottom.window.height), (16, 8)); + assert!(bottom.bands.iter().all(|band| band.len() == 16 * 8)); + assert_eq!(read_all(&mut reader), want); +} + +#[test] +fn planar_strips_round_trip() { + let case = Case { + bands: 3, + planar: true, + layout: RawLayout::Strips { rows_per_strip: 5 }, + predictor: Pred::None, + compression: NONE, + }; + let (bytes, want) = encode::(&case, 8, 1); + let mut reader = open(&bytes).expect("opens"); + assert_eq!(reader.layout().kind, BlockKind::Strips); + assert_eq!(reader.block_count(), H.div_ceil(5)); + assert_eq!(read_all(&mut reader), want); +} + +#[test] +fn tiles_with_horizontal_predictor_round_trip() { + let case = Case { + bands: 2, + planar: false, + layout: TILE, + predictor: Pred::Horizontal, + compression: DEFLATE, + }; + let (bytes, want) = encode::(&case, 16, 1); + let mut reader = open(&bytes).expect("opens"); + assert_eq!(read_all(&mut reader), want); +} + +#[test] +fn planar_float_tiles_with_floating_point_predictor_round_trip() { + let case = Case { + bands: 2, + planar: true, + layout: TILE, + predictor: Pred::FloatingPoint, + compression: DEFLATE, + }; + let (bytes, want) = encode::(&case, 32, 3); + let mut reader = open(&bytes).expect("opens"); + assert_eq!(reader.sample_type(), SampleType::F32); + assert_eq!(read_all(&mut reader), want); +} + +#[test] +fn u32_and_f64_tiles_round_trip() { + let case = Case { + bands: 1, + planar: false, + layout: TILE, + predictor: Pred::None, + compression: LZW, + }; + let (bytes, want) = encode::(&case, 32, 1); + let mut reader = open(&bytes).expect("opens"); + assert_eq!(reader.sample_type(), SampleType::U32); + assert_eq!(read_all(&mut reader), want); + + let (bytes, want) = encode::(&case, 64, 3); + let mut reader = open(&bytes).expect("opens"); + assert_eq!(reader.sample_type(), SampleType::F64); + assert_eq!(read_all(&mut reader), want); +} diff --git a/nodedb-spatial/src/lib.rs b/nodedb-spatial/src/lib.rs index bd7726cac..a68490a25 100644 --- a/nodedb-spatial/src/lib.rs +++ b/nodedb-spatial/src/lib.rs @@ -18,6 +18,7 @@ pub mod h3; pub mod hybrid; pub mod measures; pub mod operations; +pub mod orient; pub mod persist; pub mod planar; pub mod predicates; @@ -39,6 +40,7 @@ pub use h3::{ pub use hybrid::{SpatialPreFilterResult, bitmap_contains, ids_to_bitmap, spatial_prefilter}; pub use measures::{st_area, st_centroid, st_centroid_in}; pub use operations::{st_buffer, st_buffer_in, st_envelope, st_envelope_in, st_union}; +pub use orient::orient_rings; pub use persist::{ RTreeCheckpointError, SpatialIndexMeta, SpatialIndexType, deserialize_meta, meta_storage_key, rtree_storage_key, serialize_meta, @@ -52,4 +54,4 @@ pub use predicates::{ pub use rtree::{RTree, RTreeEntry, SridMismatch}; pub use validate::{is_valid, validate_geometry}; pub use wkb::{geometry_from_wkb, geometry_to_ewkb, geometry_to_wkb}; -pub use wkt::{geometry_from_wkt, geometry_to_wkt}; +pub use wkt::{Ewkt, geometry_from_ewkt, geometry_from_wkt, geometry_to_wkt}; diff --git a/nodedb-spatial/src/orient.rs b/nodedb-spatial/src/orient.rs new file mode 100644 index 000000000..6e284a95e --- /dev/null +++ b/nodedb-spatial/src/orient.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Ring orientation by the right-hand rule: exterior rings counter-clockwise, +//! holes clockwise. +//! +//! A caller whose use of a polygon does not depend on ring direction (an +//! even-odd point-in-polygon test) orients the rings first, so a polygon +//! drawn clockwise, as shapefiles store it, is accepted. + +use nodedb_types::geometry::Geometry; + +use crate::validate::signed_area; + +/// Reverse every ring of `geom` that breaks the right-hand rule. Polygons +/// nested in a `MultiPolygon` or `GeometryCollection` are oriented too. +/// Other geometry types are unchanged. +pub fn orient_rings(geom: &mut Geometry) { + match geom { + Geometry::Polygon { coordinates } => orient_polygon(coordinates), + Geometry::MultiPolygon { coordinates } => { + coordinates.iter_mut().for_each(|p| orient_polygon(p)); + } + Geometry::GeometryCollection { geometries } => { + geometries.iter_mut().for_each(orient_rings); + } + _ => {} + } +} + +fn orient_polygon(rings: &mut [Vec<[f64; 2]>]) { + for (index, ring) in rings.iter_mut().enumerate() { + let area = signed_area(ring); + let wrong = if index == 0 { area < 0.0 } else { area > 0.0 }; + if wrong { + ring.reverse(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::validate::validate_geometry; + + fn square(clockwise: bool, lo: f64, hi: f64) -> Vec<[f64; 2]> { + let mut ring = vec![[lo, lo], [hi, lo], [hi, hi], [lo, hi], [lo, lo]]; + if clockwise { + ring.reverse(); + } + ring + } + + #[test] + fn clockwise_exterior_and_ccw_hole_become_valid() { + let mut geom = Geometry::polygon(vec![square(true, 0.0, 10.0), square(false, 2.0, 4.0)]); + assert!(!validate_geometry(&geom).is_empty()); + orient_rings(&mut geom); + assert!(validate_geometry(&geom).is_empty(), "{geom:?}"); + } + + #[test] + fn oriented_polygon_is_unchanged() { + let original = Geometry::polygon(vec![square(false, 0.0, 10.0), square(true, 2.0, 4.0)]); + let mut geom = original.clone(); + orient_rings(&mut geom); + assert_eq!(geom, original); + } + + #[test] + fn multipolygon_members_are_oriented() { + let mut geom = Geometry::MultiPolygon { + coordinates: vec![vec![square(true, 0.0, 1.0)], vec![square(false, 5.0, 6.0)]], + }; + orient_rings(&mut geom); + assert!(validate_geometry(&geom).is_empty(), "{geom:?}"); + } +} diff --git a/nodedb-spatial/src/validate.rs b/nodedb-spatial/src/validate.rs index d34105a88..2c6816eeb 100644 --- a/nodedb-spatial/src/validate.rs +++ b/nodedb-spatial/src/validate.rs @@ -135,7 +135,7 @@ fn validate_recursive(geom: &Geometry, issues: &mut Vec) { } /// Signed area of a ring (Shoelace formula). Positive = CCW, Negative = CW. -fn signed_area(ring: &[[f64; 2]]) -> f64 { +pub(crate) fn signed_area(ring: &[[f64; 2]]) -> f64 { let n = ring.len(); if n < 3 { return 0.0; diff --git a/nodedb-spatial/src/wkt.rs b/nodedb-spatial/src/wkt.rs index cb869198b..4cd943847 100644 --- a/nodedb-spatial/src/wkt.rs +++ b/nodedb-spatial/src/wkt.rs @@ -89,6 +89,39 @@ pub fn geometry_from_wkt(input: &str) -> Option { } } +/// EWKT text split into its optional SRID code and its geometry. +#[derive(Debug, Clone, PartialEq)] +pub struct Ewkt { + /// The code after `SRID=`, unresolved. `None` when the text has no prefix. + pub srid: Option, + pub geometry: Geometry, +} + +/// Parse EWKT: an optional `SRID=;` prefix, then WKT. +/// +/// The prefix matches case-insensitively. The code is ASCII digits only. +/// Returns `None` for a malformed prefix or malformed WKT. +pub fn geometry_from_ewkt(input: &str) -> Option { + let s = input.trim_start(); + let Some(rest) = strip_prefix_ci(s, "SRID=") else { + return geometry_from_wkt(s).map(|geometry| Ewkt { + srid: None, + geometry, + }); + }; + let (code, wkt) = rest.split_once(';')?; + let code = code.trim(); + if code.is_empty() || !code.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let srid = code.parse::().ok()?; + let geometry = geometry_from_wkt(wkt)?; + Some(Ewkt { + srid: Some(srid), + geometry, + }) +} + // ── Serialization helpers ── fn coords_to_wkt(coords: &[[f64; 2]]) -> String { @@ -101,10 +134,13 @@ fn coords_to_wkt(coords: &[[f64; 2]]) -> String { // ── Parsing helpers ── -/// Case-insensitive prefix strip. +/// Case-insensitive prefix strip of an ASCII `prefix`. +/// +/// Compares bytes, so a multi-byte character at the cut point never panics. fn strip_prefix_ci<'a>(s: &'a str, prefix: &str) -> Option<&'a str> { - if s.len() >= prefix.len() && s[..prefix.len()].eq_ignore_ascii_case(prefix) { - Some(&s[prefix.len()..]) + let head = s.as_bytes().get(..prefix.len())?; + if head.eq_ignore_ascii_case(prefix.as_bytes()) { + s.get(prefix.len()..) } else { None } @@ -369,6 +405,43 @@ mod tests { assert_eq!(parsed, Geometry::point(5.0, 10.0)); } + #[test] + fn ewkt_prefix_is_case_insensitive() { + let square = Geometry::polygon(vec![vec![[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 0.0]]]); + for text in [ + "SRID=4326;POLYGON((0 0, 1 0, 1 1, 0 0))", + "srid=4326;polygon((0 0, 1 0, 1 1, 0 0))", + " SrId= 4326 ;POLYGON((0 0, 1 0, 1 1, 0 0))", + ] { + let parsed = geometry_from_ewkt(text).unwrap(); + assert_eq!(parsed.srid, Some(4326), "{text}"); + assert_eq!(parsed.geometry, square, "{text}"); + } + } + + #[test] + fn ewkt_without_prefix_has_no_srid() { + let parsed = geometry_from_ewkt("POINT(5 10)").unwrap(); + assert_eq!(parsed.srid, None); + assert_eq!(parsed.geometry, Geometry::point(5.0, 10.0)); + } + + #[test] + fn malformed_ewkt_returns_none() { + assert!(geometry_from_ewkt("SRID=;POINT(1 2)").is_none()); + assert!(geometry_from_ewkt("SRID=43x6;POINT(1 2)").is_none()); + assert!(geometry_from_ewkt("SRID=-4326;POINT(1 2)").is_none()); + assert!(geometry_from_ewkt("SRID=4326 POINT(1 2)").is_none()); + assert!(geometry_from_ewkt("SRID=4326;GARBAGE").is_none()); + } + + #[test] + fn a_multibyte_character_at_the_prefix_cut_does_not_panic() { + assert!(geometry_from_ewkt("SRÍD=4326;POINT(1 2)").is_none()); + assert!(geometry_from_wkt("POIÑT(1 2)").is_none()); + assert!(geometry_from_wkt("é").is_none()); + } + #[test] fn invalid_wkt_returns_none() { assert!(geometry_from_wkt("").is_none()); diff --git a/nodedb-sql/src/catalog.rs b/nodedb-sql/src/catalog.rs index ff79ef02d..3fbcb23ff 100644 --- a/nodedb-sql/src/catalog.rs +++ b/nodedb-sql/src/catalog.rs @@ -6,7 +6,7 @@ use nodedb_types::DatabaseId; use thiserror::Error; use crate::types::CollectionInfo; -use crate::types_array::{ArrayAttrAst, ArrayDimAst}; +use crate::types_array::{ArrayAttrAst, ArrayDimAst, ArrayGeorefAst}; /// Normalize the PostgreSQL identifier spelling accepted by `regclass` input. /// @@ -273,4 +273,6 @@ pub struct ArrayCatalogView { pub dims: Vec, pub attrs: Vec, pub tile_extents: Vec, + /// The array's georef, with its axes named by dim. `None` = no georef. + pub georef: Option, } diff --git a/nodedb-sql/src/ddl_ast/parse/copy_from.rs b/nodedb-sql/src/ddl_ast/parse/copy_from.rs index 9926d0395..55a5127ea 100644 --- a/nodedb-sql/src/ddl_ast/parse/copy_from.rs +++ b/nodedb-sql/src/ddl_ast/parse/copy_from.rs @@ -1,8 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 -//! Parse `COPY FROM '' [WITH (FORMAT ..., DELIMITER ..., HEADER ...)]`. +//! Parse `COPY FROM '' [WITH (...)]`. //! -//! The server-side file-path form parses here. The STDIN streaming shape +//! The server-side file-path form parses here. Its options parse in +//! `copy_options`. The STDIN streaming shape //! (`COPY FROM STDIN`) parses in `copy_from_stdin`. The restore shape //! `COPY tenant_restore() FROM STDIN` returns `None`, so dispatch falls //! through to the backup/restore handler. `COPY ... TO` and @@ -12,7 +13,8 @@ //! `FROM` or `TO` inside a path, a name, or a query. use super::clause::{ClauseCursor, Tok, Token, tokenize}; -use crate::ddl_ast::statement::{CopyFormat, MiscStmt, NodedbStatement}; +use super::copy_options::{detect_format_from_path, parse_copy_options}; +use crate::ddl_ast::statement::{MiscStmt, NodedbStatement}; use crate::error::SqlError; /// Which way a `COPY` statement moves rows. @@ -110,17 +112,19 @@ fn parse_copy_from(sql: &str) -> Result { let path = c .string("FROM") .map_err(|_| c.error("expected single-quoted path after FROM"))?; - let (format, delimiter, header) = parse_copy_options(&mut c)?; - let format = match format { + let options = parse_copy_options(&mut c)?; + let format = match options.format.clone() { Some(f) => Some(f), None => detect_format_from_path(&path)?, }; + options.check_for(format.as_ref())?; Ok(NodedbStatement::Misc(MiscStmt::CopyFromFile { collection, path, format, - delimiter, - header, + delimiter: options.delimiter, + header: options.header.unwrap_or(true), + raster: options.raster, })) } @@ -135,103 +139,11 @@ pub(super) fn copy_collection_name(c: &mut ClauseCursor<'_, '_>) -> Result, -) -> Result<(Option, Option, bool), SqlError> { - let mut format = None; - let mut delimiter = None; - let mut header = true; - if c.eat_keyword("WITH") { - let options = c.paren_list("WITH options", |c| { - let key = c.word("option name")?.to_ascii_uppercase(); - c.eat_punct('='); - let value = match c.string(&key) { - Ok(value) => value, - Err(_) => c.word(&key)?.to_string(), - }; - Ok((key, value)) - })?; - for (key, value) in options { - match key.as_str() { - "FORMAT" => format = Some(parse_format_value(&value)?), - "DELIMITER" => delimiter = Some(parse_delimiter_value(&value)?), - "HEADER" => { - header = parse_bool_value(&value).ok_or_else(|| { - c.error(format!("HEADER must be true or false — found '{value}'")) - })?; - } - other => return Err(c.error(format!("WITH: unknown option '{other}'"))), - } - } - } - if !c.at_end() { - return Err(c.error(format!("unexpected trailing content {}", c.describe_next()))); - } - Ok((format, delimiter, header)) -} - -fn parse_format_value(s: &str) -> Result { - match s.trim().to_lowercase().as_str() { - "ndjson" | "jsonl" => Ok(CopyFormat::Ndjson), - "json" => Ok(CopyFormat::JsonArray), - "csv" => Ok(CopyFormat::Csv), - other => Err(SqlError::Parse { - detail: format!("COPY: unknown FORMAT '{other}'; expected ndjson, json, or csv"), - }), - } -} - -pub(super) fn parse_delimiter_value(s: &str) -> Result { - let mut chars = s.chars(); - let ch = chars.next().ok_or_else(|| SqlError::Parse { - detail: "COPY: DELIMITER must be a single character".to_string(), - })?; - if chars.next().is_some() { - return Err(SqlError::Parse { - detail: "COPY: DELIMITER must be a single character".to_string(), - }); - } - Ok(ch) -} - -pub(super) fn parse_bool_value(s: &str) -> Option { - match s.trim().to_lowercase().as_str() { - "true" | "1" | "yes" | "on" => Some(true), - "false" | "0" | "no" | "off" => Some(false), - _ => None, - } -} - -/// Detect the copy format from the file path extension. -/// -/// Returns `Err` if the extension is unrecognised or absent (suggests an -/// explicit WITH clause). -pub(super) fn detect_format_from_path(path: &str) -> Result, SqlError> { - let lower = path.to_lowercase(); - if lower.ends_with(".ndjson") || lower.ends_with(".jsonl") { - return Ok(Some(CopyFormat::Ndjson)); - } - if lower.ends_with(".json") { - return Ok(Some(CopyFormat::JsonArray)); - } - if lower.ends_with(".csv") { - return Ok(Some(CopyFormat::Csv)); - } - Err(SqlError::Parse { - detail: format!( - "COPY: cannot infer format from path '{path}'; \ - add WITH (FORMAT ndjson|json|csv)" - ), - }) -} - #[cfg(test)] mod tests { use super::*; use crate::ddl_ast::parse::dispatch::parse; + use crate::ddl_ast::statement::CopyFormat; fn ok(sql: &str) -> NodedbStatement { parse(sql).expect("expected Some").expect("expected Ok") @@ -267,12 +179,14 @@ mod tests { format, delimiter, header, + raster, }) => { assert_eq!(collection, "users"); assert_eq!(path, "/tmp/users.ndjson"); assert_eq!(format, Some(CopyFormat::Ndjson)); assert_eq!(delimiter, None); assert!(header); + assert!(raster.is_empty()); } other => panic!("unexpected: {other:?}"), } diff --git a/nodedb-sql/src/ddl_ast/parse/copy_from_stdin.rs b/nodedb-sql/src/ddl_ast/parse/copy_from_stdin.rs index 3ad96d259..f5b1ae100 100644 --- a/nodedb-sql/src/ddl_ast/parse/copy_from_stdin.rs +++ b/nodedb-sql/src/ddl_ast/parse/copy_from_stdin.rs @@ -7,15 +7,17 @@ //! STDIN` is the restore shape, which the backup detector owns, so it //! returns `None`. -use super::clause::{ClauseCursor, Tok, tokenize}; -use super::copy_from::{copy_collection_name, parse_bool_value, parse_delimiter_value}; +use super::clause::{ClauseCursor, Tok, Token, tokenize}; +use super::copy_from::copy_collection_name; +use super::copy_options::{parse_bool_value, parse_delimiter_value}; use crate::ddl_ast::statement::{CopyStdinFormat, MiscStmt, NodedbStatement}; use crate::error::SqlError; /// Parse a `COPY ... FROM STDIN` statement. `None` when the copied relation /// is more than one name token, or the options are not this grammar's (a /// `FORMAT json`, say): such a statement keeps the refusal the bulk-import -/// router gives it. +/// router gives it. `FORMAT geotiff` is refused here with `0A000`, naming +/// the server-path form. pub(super) fn try_parse(sql: &str) -> Option> { let toks = tokenize(sql); // `COPY`, the name, then `FROM`. @@ -26,10 +28,35 @@ pub(super) fn try_parse(sql: &str) -> Option> if !one_name { return None; } + // Checked before the options parse, so any other option in the list + // still gets this refusal. + if names_geotiff(&toks) { + return Some(Err(SqlError::FeatureNotSupported { + detail: "COPY FROM STDIN does not read GeoTIFF; load the file from a server \ + path with COPY FROM '.tif' WITH (FORMAT geotiff)" + .to_string(), + })); + } let mut c = ClauseCursor::new(sql, &toks, "COPY"); parse_stdin(&mut c).ok().map(Ok) } +/// True when the tokens hold `FORMAT [=] geotiff`, bare or quoted. +fn names_geotiff(toks: &[Token<'_>]) -> bool { + let is_geotiff = |tok: Option<&Token<'_>>| match tok.map(|t| &t.tok) { + Some(Tok::Word(w)) => w.eq_ignore_ascii_case("geotiff"), + Some(Tok::Str(s)) => s.trim().eq_ignore_ascii_case("geotiff"), + _ => false, + }; + toks.iter().enumerate().any(|(i, t)| { + matches!(&t.tok, Tok::Word(w) if w.eq_ignore_ascii_case("FORMAT")) + && match toks.get(i + 1).map(|t| &t.tok) { + Some(Tok::Punct('=')) => is_geotiff(toks.get(i + 2)), + _ => is_geotiff(toks.get(i + 1)), + } + }) +} + fn parse_stdin(c: &mut ClauseCursor<'_, '_>) -> Result { c.expect_keyword("COPY")?; let collection = copy_collection_name(c)?; @@ -140,6 +167,25 @@ mod tests { assert!(parse("COPY scene FROM STDIN WITH (BOGUS 1)").is_none()); } + #[test] + fn geotiff_over_stdin_is_refused_naming_the_server_path_form() { + for sql in [ + "COPY scene FROM STDIN WITH (FORMAT geotiff)", + "COPY scene FROM STDIN WITH (FORMAT 'GeoTIFF', HEADER)", + "COPY scene FROM STDIN WITH (BANDS (a, b), FORMAT = geotiff)", + ] { + let refusal = parse(sql) + .expect("expected Some") + .expect_err("expected Err"); + match refusal { + SqlError::FeatureNotSupported { detail } => { + assert!(detail.contains("FROM '.tif'"), "{detail}"); + } + other => panic!("expected FeatureNotSupported, got {other:?}"), + } + } + } + #[test] fn restore_shape_is_not_parsed() { assert!(parse("COPY tenant_restore(1) FROM STDIN").is_none()); diff --git a/nodedb-sql/src/ddl_ast/parse/copy_options.rs b/nodedb-sql/src/ddl_ast/parse/copy_options.rs new file mode 100644 index 000000000..ea4571314 --- /dev/null +++ b/nodedb-sql/src/ddl_ast/parse/copy_options.rs @@ -0,0 +1,400 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The `WITH (...)` options of a server-path `COPY`, and format detection +//! from the file extension. +//! +//! Text formats take `FORMAT`, `DELIMITER` and `HEADER`. GeoTIFF takes +//! `FORMAT geotiff`, `BANDS (name | SKIP, ...)`, `NODATA ` and +//! `TIME ''`. An option of the other family is refused once the +//! format is known. + +use nodedb_types::NdbDateTime; + +use super::clause::{ClauseCursor, Tok, Token, parse_err}; +use crate::ddl_ast::statement::{CopyBand, CopyFormat, CopyRasterOptions}; +use crate::error::SqlError; + +/// The options of one `COPY ... WITH (...)` clause, before the format is +/// resolved. +#[derive(Debug, Default)] +pub(super) struct CopyOptions { + pub format: Option, + pub delimiter: Option, + /// `None` when `HEADER` is absent: each format picks its own default. + pub header: Option, + pub raster: CopyRasterOptions, +} + +impl CopyOptions { + /// Refuse options that do not apply to `format`. + pub(super) fn check_for(&self, format: Option<&CopyFormat>) -> Result<(), SqlError> { + let geotiff = format == Some(&CopyFormat::GeoTiff); + if geotiff && (self.delimiter.is_some() || self.header.is_some()) { + return Err(parse_err( + "COPY: DELIMITER and HEADER apply to text formats, not FORMAT geotiff", + )); + } + if !geotiff && !self.raster.is_empty() { + return Err(parse_err( + "COPY: BANDS, NODATA and TIME need FORMAT geotiff (or a .tif / .tiff path)", + )); + } + Ok(()) + } +} + +/// One option value, typed by its key. +enum OptionValue { + Text(String), + Bands(Vec), + Nodata(f64), + TimeMs(i64), +} + +/// Read an optional `WITH (key [=] value, ...)` and require the end of the +/// statement. +pub(super) fn parse_copy_options(c: &mut ClauseCursor<'_, '_>) -> Result { + let mut options = CopyOptions::default(); + if c.eat_keyword("WITH") { + let entries = c.paren_list("WITH options", |c| { + let key = c.word("option name")?.to_ascii_uppercase(); + c.eat_punct('='); + let value = match key.as_str() { + "BANDS" => OptionValue::Bands(bands_value(c)?), + "NODATA" => OptionValue::Nodata(nodata_value(c)?), + "TIME" => OptionValue::TimeMs(time_value(c)?), + _ => OptionValue::Text(match c.string(&key) { + Ok(value) => value, + Err(_) => c.word(&key)?.to_string(), + }), + }; + Ok((key, value)) + })?; + let mut seen: Vec = Vec::with_capacity(entries.len()); + for (key, value) in entries { + if seen.contains(&key) { + return Err(c.error(format!("WITH: option '{key}' given twice"))); + } + apply(c, &mut options, &key, value)?; + seen.push(key); + } + } + if !c.at_end() { + return Err(c.error(format!("unexpected trailing content {}", c.describe_next()))); + } + Ok(options) +} + +fn apply( + c: &ClauseCursor<'_, '_>, + options: &mut CopyOptions, + key: &str, + value: OptionValue, +) -> Result<(), SqlError> { + match (key, value) { + ("FORMAT", OptionValue::Text(text)) => options.format = Some(parse_format_value(&text)?), + ("DELIMITER", OptionValue::Text(text)) => { + options.delimiter = Some(parse_delimiter_value(&text)?); + } + ("HEADER", OptionValue::Text(text)) => { + let header = parse_bool_value(&text) + .ok_or_else(|| c.error(format!("HEADER must be true or false — found '{text}'")))?; + options.header = Some(header); + } + (_, OptionValue::Bands(bands)) => options.raster.bands = Some(bands), + (_, OptionValue::Nodata(nodata)) => options.raster.nodata = Some(nodata), + (_, OptionValue::TimeMs(time_ms)) => options.raster.time_ms = Some(time_ms), + (other, OptionValue::Text(_)) => { + return Err(c.error(format!("WITH: unknown option '{other}'"))); + } + } + Ok(()) +} + +/// `( name | SKIP , ... )` with at least one name and no name twice. +fn bands_value(c: &mut ClauseCursor<'_, '_>) -> Result, SqlError> { + let bands = c.paren_list("BANDS", |c| { + if c.eat_keyword("SKIP") { + Ok(CopyBand::Skip) + } else { + c.name("BANDS entry").map(CopyBand::Attr) + } + })?; + let mut names: Vec<&str> = Vec::with_capacity(bands.len()); + for band in &bands { + if let CopyBand::Attr(name) = band { + if names.contains(&name.as_str()) { + return Err(c.error(format!("BANDS names attribute '{name}' twice"))); + } + names.push(name); + } + } + if names.is_empty() { + return Err(c.error("BANDS must load at least one band; every entry is SKIP")); + } + Ok(bands) +} + +/// A number with an optional leading `-`. The tokenizer splits an exponent +/// sign, so `1e-5` arrives as `1e`, `-`, `5` and is joined back here. +fn nodata_value(c: &mut ClauseCursor<'_, '_>) -> Result { + let toks = c.remaining_tokens(); + let mut text = String::new(); + let mut used = 0; + if matches!( + toks.first(), + Some(Token { + tok: Tok::Punct('-'), + .. + }) + ) { + text.push('-'); + used = 1; + } + let Some(Token { + tok: Tok::Word(mantissa), + span, + }) = toks.get(used) + else { + return Err(c.error(format!( + "NODATA needs a number — found {}", + c.describe_next() + ))); + }; + text.push_str(mantissa); + used += 1; + if mantissa.ends_with(['e', 'E']) + && let ( + Some(Token { + tok: Tok::Punct(sign @ ('+' | '-')), + span: sign_span, + }), + Some(Token { + tok: Tok::Word(exponent), + span: exponent_span, + }), + ) = (toks.get(used), toks.get(used + 1)) + && sign_span.start == span.end + && exponent_span.start == sign_span.end + { + text.push(*sign); + text.push_str(exponent); + used += 2; + } + let value = text + .parse::() + .map_err(|_| c.error(format!("NODATA must be a number — found '{text}'")))?; + c.skip(used); + Ok(value) +} + +/// A quoted ISO 8601 instant, in milliseconds since the Unix epoch. +fn time_value(c: &mut ClauseCursor<'_, '_>) -> Result { + let text = c.string("TIME")?; + NdbDateTime::parse(&text) + .map(|instant| instant.unix_millis()) + .ok_or_else(|| { + c.error(format!( + "TIME must be an ISO 8601 timestamp such as '2024-03-15T10:30:00Z' — found '{text}'" + )) + }) +} + +pub(super) fn parse_format_value(s: &str) -> Result { + match s.trim().to_lowercase().as_str() { + "ndjson" | "jsonl" => Ok(CopyFormat::Ndjson), + "json" => Ok(CopyFormat::JsonArray), + "csv" => Ok(CopyFormat::Csv), + "geotiff" => Ok(CopyFormat::GeoTiff), + other => Err(SqlError::Parse { + detail: format!( + "COPY: unknown FORMAT '{other}'; expected ndjson, json, csv, or geotiff" + ), + }), + } +} + +pub(super) fn parse_delimiter_value(s: &str) -> Result { + let mut chars = s.chars(); + let ch = chars.next().ok_or_else(|| SqlError::Parse { + detail: "COPY: DELIMITER must be a single character".to_string(), + })?; + if chars.next().is_some() { + return Err(SqlError::Parse { + detail: "COPY: DELIMITER must be a single character".to_string(), + }); + } + Ok(ch) +} + +pub(super) fn parse_bool_value(s: &str) -> Option { + match s.trim().to_lowercase().as_str() { + "true" | "1" | "yes" | "on" => Some(true), + "false" | "0" | "no" | "off" => Some(false), + _ => None, + } +} + +/// Detect the copy format from the file path extension. +/// +/// Returns `Err` if the extension is unrecognised or absent (suggests an +/// explicit WITH clause). +pub(super) fn detect_format_from_path(path: &str) -> Result, SqlError> { + let lower = path.to_lowercase(); + if lower.ends_with(".ndjson") || lower.ends_with(".jsonl") { + return Ok(Some(CopyFormat::Ndjson)); + } + if lower.ends_with(".json") { + return Ok(Some(CopyFormat::JsonArray)); + } + if lower.ends_with(".csv") { + return Ok(Some(CopyFormat::Csv)); + } + if lower.ends_with(".tif") || lower.ends_with(".tiff") { + return Ok(Some(CopyFormat::GeoTiff)); + } + Err(SqlError::Parse { + detail: format!( + "COPY: cannot infer format from path '{path}'; \ + add WITH (FORMAT ndjson|json|csv|geotiff)" + ), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ddl_ast::parse::dispatch::parse; + use crate::ddl_ast::statement::{MiscStmt, NodedbStatement}; + + fn copy_from(sql: &str) -> (Option, CopyRasterOptions) { + match parse(sql).expect("expected Some").expect("expected Ok") { + NodedbStatement::Misc(MiscStmt::CopyFromFile { format, raster, .. }) => { + (format, raster) + } + other => panic!("unexpected: {other:?}"), + } + } + + fn refusal(sql: &str) -> String { + parse(sql) + .expect("expected Some") + .expect_err("expected Err") + .to_string() + } + + #[test] + fn geotiff_format_parses_explicitly() { + let (format, raster) = copy_from("COPY scene FROM '/data/scene.bin' WITH (FORMAT geotiff)"); + assert_eq!(format, Some(CopyFormat::GeoTiff)); + assert!(raster.is_empty()); + } + + #[test] + fn tif_and_tiff_paths_detect_geotiff() { + for path in ["/data/a.tif", "/data/b.TIFF", "/data/c.Tif"] { + let (format, _) = copy_from(&format!("COPY scene FROM '{path}'")); + assert_eq!(format, Some(CopyFormat::GeoTiff), "{path}"); + } + } + + #[test] + fn bands_take_names_and_skip() { + let (_, raster) = copy_from( + "COPY scene FROM '/d/s.tif' WITH (FORMAT geotiff, BANDS (red, SKIP, \"NIR\"))", + ); + assert_eq!( + raster.bands, + Some(vec![ + CopyBand::Attr("red".into()), + CopyBand::Skip, + CopyBand::Attr("NIR".into()), + ]) + ); + } + + #[test] + fn bands_refuse_duplicates_and_all_skip() { + assert!(refusal("COPY s FROM '/d/s.tif' WITH (BANDS (a, b, a))").contains("twice")); + assert!(refusal("COPY s FROM '/d/s.tif' WITH (BANDS (SKIP, skip))").contains("SKIP")); + assert!(refusal("COPY s FROM '/d/s.tif' WITH (BANDS ())").contains("empty")); + } + + #[test] + fn nodata_takes_signed_and_exponent_numbers() { + let cases = [ + ("NODATA 0", 0.0), + ("NODATA -9999", -9999.0), + ("NODATA = 255", 255.0), + ("NODATA -3.4028235e+38", -3.402_823_5e38), + ("NODATA 1e-5", 1e-5), + ("NODATA 2.5E3", 2500.0), + ]; + for (option, want) in cases { + let (_, raster) = copy_from(&format!("COPY s FROM '/d/s.tif' WITH ({option})")); + assert_eq!(raster.nodata, Some(want), "{option}"); + } + let (_, raster) = copy_from("COPY s FROM '/d/s.tif' WITH (NODATA NaN)"); + assert!(raster.nodata.is_some_and(f64::is_nan)); + } + + #[test] + fn nodata_refuses_non_numbers() { + assert!(refusal("COPY s FROM '/d/s.tif' WITH (NODATA abc)").contains("NODATA")); + assert!(refusal("COPY s FROM '/d/s.tif' WITH (NODATA '5')").contains("NODATA")); + assert!(refusal("COPY s FROM '/d/s.tif' WITH (NODATA)").contains("NODATA")); + } + + #[test] + fn time_parses_iso_8601_to_millis() { + let (_, raster) = + copy_from("COPY s FROM '/d/s.tif' WITH (TIME '2024-03-15T10:30:00.250Z')"); + assert_eq!(raster.time_ms, Some(1_710_498_600_250)); + let (_, raster) = copy_from("COPY s FROM '/d/s.tif' WITH (TIME '2024-03-15')"); + assert_eq!(raster.time_ms, Some(1_710_460_800_000)); + assert!(refusal("COPY s FROM '/d/s.tif' WITH (TIME 'yesterday')").contains("ISO 8601")); + assert!(refusal("COPY s FROM '/d/s.tif' WITH (TIME 2024)").contains("TIME")); + } + + #[test] + fn every_raster_option_together() { + let (format, raster) = copy_from( + "COPY s FROM '/d/s.tiff' WITH (FORMAT geotiff, BANDS (b1, b2), NODATA -1, \ + TIME '2024-01-01T00:00:00Z')", + ); + assert_eq!(format, Some(CopyFormat::GeoTiff)); + assert_eq!(raster.bands.as_ref().map(Vec::len), Some(2)); + assert_eq!(raster.nodata, Some(-1.0)); + assert_eq!(raster.time_ms, Some(1_704_067_200_000)); + } + + #[test] + fn raster_options_need_geotiff() { + let message = refusal("COPY s FROM '/d/s.csv' WITH (NODATA 0)"); + assert!(message.contains("FORMAT geotiff"), "{message}"); + let message = refusal("COPY s FROM '/d/s.tif' WITH (FORMAT csv, BANDS (a))"); + assert!(message.contains("FORMAT geotiff"), "{message}"); + } + + #[test] + fn text_options_are_refused_for_geotiff() { + let message = refusal("COPY s FROM '/d/s.tif' WITH (HEADER true)"); + assert!(message.contains("text formats"), "{message}"); + let message = refusal("COPY s FROM '/d/s.bin' WITH (FORMAT geotiff, DELIMITER ';')"); + assert!(message.contains("text formats"), "{message}"); + } + + #[test] + fn an_option_given_twice_is_refused() { + let message = refusal("COPY s FROM '/d/s.tif' WITH (NODATA 1, NODATA 2)"); + assert!(message.contains("twice"), "{message}"); + let message = refusal("COPY s FROM '/d/s.csv' WITH (FORMAT csv, FORMAT json)"); + assert!(message.contains("twice"), "{message}"); + } + + #[test] + fn unknown_format_lists_geotiff() { + let message = refusal("COPY s FROM '/d/s.csv' WITH (FORMAT parquet)"); + assert!(message.contains("geotiff"), "{message}"); + } +} diff --git a/nodedb-sql/src/ddl_ast/parse/copy_to.rs b/nodedb-sql/src/ddl_ast/parse/copy_to.rs index c907b9007..9b9d478ae 100644 --- a/nodedb-sql/src/ddl_ast/parse/copy_to.rs +++ b/nodedb-sql/src/ddl_ast/parse/copy_to.rs @@ -3,14 +3,12 @@ //! Parse `COPY TO '' [WITH (...)]` and //! `COPY (SELECT ...) TO '' [WITH (...)]`. -use crate::ddl_ast::statement::{CopyToSource, MiscStmt, NodedbStatement}; +use crate::ddl_ast::statement::{CopyFormat, CopyToSource, MiscStmt, NodedbStatement}; use crate::error::SqlError; use super::clause::{ClauseCursor, tokenize}; -use super::copy_from::{ - CopyDirection, copy_collection_name, copy_direction, detect_format_from_path, - parse_copy_options, -}; +use super::copy_from::{CopyDirection, copy_collection_name, copy_direction}; +use super::copy_options::{detect_format_from_path, parse_copy_options}; /// Try to parse a COPY TO statement. /// @@ -47,17 +45,24 @@ fn parse_copy_to(sql: &str) -> Result { c.describe_next() )) })?; - let (format, delimiter, header) = parse_copy_options(&mut c)?; - let format = match format { + let options = parse_copy_options(&mut c)?; + let format = match options.format.clone() { Some(f) => Some(f), None => detect_format_from_path(&path)?, }; + if format == Some(CopyFormat::GeoTiff) { + return Err(SqlError::FeatureNotSupported { + detail: "COPY TO does not write GeoTIFF; export with FORMAT ndjson, json, or csv" + .to_string(), + }); + } + options.check_for(format.as_ref())?; Ok(NodedbStatement::Misc(MiscStmt::CopyToFile { source, path, format, - delimiter, - header, + delimiter: options.delimiter, + header: options.header.unwrap_or(true), })) } @@ -183,6 +188,25 @@ mod tests { } } + #[test] + fn geotiff_export_is_refused() { + for sql in [ + "COPY users TO '/tmp/out.tif'", + "COPY users TO '/tmp/out.bin' WITH (FORMAT geotiff)", + ] { + assert!( + matches!(err(sql), SqlError::FeatureNotSupported { .. }), + "{sql}" + ); + } + } + + #[test] + fn raster_options_are_refused_for_export() { + let e = err("COPY users TO '/tmp/out.csv' WITH (NODATA 0)"); + assert!(matches!(e, SqlError::Parse { .. })); + } + #[test] fn from_inside_the_path_does_not_change_the_direction() { let stmt = ok("COPY users TO '/tmp/copy from mars.csv'"); diff --git a/nodedb-sql/src/ddl_ast/parse/mod.rs b/nodedb-sql/src/ddl_ast/parse/mod.rs index ef43801ff..670365f94 100644 --- a/nodedb-sql/src/ddl_ast/parse/mod.rs +++ b/nodedb-sql/src/ddl_ast/parse/mod.rs @@ -9,6 +9,7 @@ mod collection; mod conflict_policy; mod copy_from; mod copy_from_stdin; +mod copy_options; mod copy_to; mod custom_type; pub mod database; diff --git a/nodedb-sql/src/ddl_ast/statement/maintenance.rs b/nodedb-sql/src/ddl_ast/statement/maintenance.rs index 31b687367..4e61ccd72 100644 --- a/nodedb-sql/src/ddl_ast/statement/maintenance.rs +++ b/nodedb-sql/src/ddl_ast/statement/maintenance.rs @@ -29,4 +29,38 @@ pub enum CopyFormat { JsonArray, /// CSV with an optional header row (`.csv`). Csv, + /// A GeoTIFF raster (`.tif` / `.tiff`), read from a server path into an + /// array. + GeoTiff, +} + +/// One `BANDS (...)` entry of a GeoTIFF `COPY`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CopyBand { + /// The band loads into this array attribute. + Attr(String), + /// `SKIP`: the band is not loaded. + Skip, +} + +/// The GeoTIFF options of `COPY FROM '' WITH (FORMAT geotiff, ...)`. +/// Every field is `None` for the other formats. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct CopyRasterOptions { + /// `BANDS (a, SKIP, b)`: entry `i` names where band `i` loads. `None` + /// loads band `i` into attribute `i`. + pub bands: Option>, + /// `NODATA `: the nodata value, in place of the file's + /// `GDAL_NODATA`. + pub nodata: Option, + /// `TIME ''`: the scene instant every loaded cell carries, in + /// milliseconds since the Unix epoch. + pub time_ms: Option, +} + +impl CopyRasterOptions { + /// True when no GeoTIFF option is given. + pub fn is_empty(&self) -> bool { + self.bands.is_none() && self.nodata.is_none() && self.time_ms.is_none() + } } diff --git a/nodedb-sql/src/ddl_ast/statement/types/misc.rs b/nodedb-sql/src/ddl_ast/statement/types/misc.rs index 00543d715..09713d77f 100644 --- a/nodedb-sql/src/ddl_ast/statement/types/misc.rs +++ b/nodedb-sql/src/ddl_ast/statement/types/misc.rs @@ -2,7 +2,9 @@ //! Miscellaneous DDL/DML statements. -use crate::ddl_ast::statement::maintenance::{CopyFormat, CopyStdinFormat, CopyToSource}; +use crate::ddl_ast::statement::maintenance::{ + CopyFormat, CopyRasterOptions, CopyStdinFormat, CopyToSource, +}; #[derive(Debug, Clone, PartialEq)] pub enum MiscStmt { @@ -17,6 +19,7 @@ pub enum MiscStmt { // ── Bulk import ────────────────────────────────────────────── /// `COPY FROM '' [WITH (FORMAT ..., DELIMITER ..., HEADER ...)]` + /// `COPY FROM '' WITH (FORMAT geotiff [, BANDS (...)] [, NODATA n] [, TIME '..'])` /// /// Server-side file-path bulk import. `CopyFromStdin` is the streaming /// form, and `CopyToFile` the export. @@ -26,6 +29,8 @@ pub enum MiscStmt { format: Option, delimiter: Option, header: bool, + /// GeoTIFF options. Empty unless `format` is `GeoTiff`. + raster: CopyRasterOptions, }, /// `COPY FROM STDIN [WITH (FORMAT csv|text, DELIMITER '', HEADER [bool])]` diff --git a/nodedb-sql/src/error.rs b/nodedb-sql/src/error.rs index e1442fbf6..61cb6a6e1 100644 --- a/nodedb-sql/src/error.rs +++ b/nodedb-sql/src/error.rs @@ -188,6 +188,14 @@ pub enum SqlError { #[error("unsupported: {detail}")] Unsupported { detail: String }, + /// A well-formed statement asks for a feature NodeDB does not provide + /// in that form, such as GeoTIFF over `COPY ... FROM STDIN`. `detail` + /// names the supported form. + /// + /// Rendered as SQLSTATE `0A000` (feature_not_supported). + #[error("feature not supported: {detail}")] + FeatureNotSupported { detail: String }, + /// A declared column DEFAULT the server cannot evaluate to a value. /// /// The column is never omitted instead. A DEFAULT that disappears stores diff --git a/nodedb-sql/src/lib.rs b/nodedb-sql/src/lib.rs index e7071703f..2e28aad35 100644 --- a/nodedb-sql/src/lib.rs +++ b/nodedb-sql/src/lib.rs @@ -195,6 +195,6 @@ fn plan_array_statement(stmt: ArrayStatement, catalog: &dyn SqlCatalog) -> Resul ArrayStatement::Drop(d) => Ok(vec![planner::array_ddl::plan_drop_array(&d)?]), ArrayStatement::Insert(i) => planner::array_dml::plan_insert_array(&i, catalog), ArrayStatement::Delete(d) => planner::array_dml::plan_delete_array(&d, catalog), - ArrayStatement::Alter(a) => Ok(vec![planner::array_ddl::plan_alter_array(&a)?]), + ArrayStatement::Alter(a) => Ok(vec![planner::array_ddl::plan_alter_array(&a, catalog)?]), } } diff --git a/nodedb-sql/src/parser/array_stmt/ast.rs b/nodedb-sql/src/parser/array_stmt/ast.rs index 55139133c..142b44f81 100644 --- a/nodedb-sql/src/parser/array_stmt/ast.rs +++ b/nodedb-sql/src/parser/array_stmt/ast.rs @@ -3,8 +3,8 @@ //! Typed AST returned by the array-statement parser. use crate::types_array::{ - ArrayAttrAst, ArrayCellOrderAst, ArrayCoordLiteral, ArrayDimAst, ArrayInsertRow, - ArrayTileOrderAst, + ArrayAttrAst, ArrayCellOrderAst, ArrayCoordLiteral, ArrayDimAst, ArrayGeorefAlter, + ArrayGeorefAst, ArrayInsertRow, ArrayTileOrderAst, }; /// Top-level array-statement AST. One variant per surface command. @@ -17,17 +17,26 @@ pub enum ArrayStatement { Alter(AlterArrayAst), } -/// `ALTER ARRAY SET ( key = value [, key = value]* )` +/// `ALTER ARRAY SET ( key = value [, key = value]* )`, +/// `ALTER ARRAY SET GEOREF (...)`, or `ALTER ARRAY DROP GEOREF`. /// -/// Supported keys: +/// Supported SET keys: /// - `audit_retain_ms` — `NULL` or non-negative integer. /// - `minimum_audit_retain_ms` — non-negative integer. #[derive(Debug, Clone, PartialEq)] pub struct AlterArrayAst { pub name: String, + pub action: AlterArrayAction, +} + +/// The one change an `ALTER ARRAY` statement makes. +#[derive(Debug, Clone, PartialEq)] +pub enum AlterArrayAction { /// Key-value pairs from the SET clause. Values are either integer /// literals or NULL (represented as `None`). - pub set: Vec<(String, Option)>, + SetOptions(Vec<(String, Option)>), + /// `SET GEOREF (...)` or `DROP GEOREF`. + Georef(ArrayGeorefAlter), } #[derive(Debug, Clone, PartialEq)] @@ -38,6 +47,8 @@ pub struct CreateArrayAst { pub tile_extents: Vec, pub cell_order: ArrayCellOrderAst, pub tile_order: ArrayTileOrderAst, + /// Optional `GEOREF (...)` clause. + pub georef: Option, /// Number of Hilbert-prefix bits used for vShard routing. /// Accepted via optional `WITH (prefix_bits = N)` clause; default 8. pub prefix_bits: u8, diff --git a/nodedb-sql/src/parser/array_stmt/lexer.rs b/nodedb-sql/src/parser/array_stmt/lexer.rs index b896ccacf..f349504c2 100644 --- a/nodedb-sql/src/parser/array_stmt/lexer.rs +++ b/nodedb-sql/src/parser/array_stmt/lexer.rs @@ -156,16 +156,26 @@ pub fn tokenize(src: &str) -> Result, SqlError> { while j < bytes.len() && bytes[j].is_ascii_digit() { j += 1; } - // Float? Require a digit after the dot — `..` is a range token. - let is_float = j + 1 < bytes.len() + // Fraction? Require a digit after the dot — `..` is a range token. + let has_fraction = j + 1 < bytes.len() && bytes[j] == b'.' && bytes[j + 1] != b'.' && bytes[j + 1].is_ascii_digit(); - if is_float { + if has_fraction { j += 1; while j < bytes.len() && bytes[j].is_ascii_digit() { j += 1; } + } + // Exponent? `e` or `E`, an optional sign, then at least one digit. + let exponent_digits_at = exponent_digits_start(bytes, j); + if let Some(digits_at) = exponent_digits_at { + j = digits_at; + while j < bytes.len() && bytes[j].is_ascii_digit() { + j += 1; + } + } + if has_fraction || exponent_digits_at.is_some() { let txt = &src[start..j]; let f: f64 = txt.parse().map_err(|_| SqlError::Parse { detail: format!("invalid float literal '{txt}'"), @@ -217,6 +227,23 @@ pub fn tokenize(src: &str) -> Result, SqlError> { Ok(out) } +/// Offset of the first exponent digit when an exponent starts at `at`. +/// +/// An exponent is `e` or `E`, an optional `+` or `-`, then a digit. +fn exponent_digits_start(bytes: &[u8], at: usize) -> Option { + if !matches!(bytes.get(at), Some(b'e' | b'E')) { + return None; + } + let digits_at = match bytes.get(at + 1) { + Some(b'+' | b'-') => at + 2, + _ => at + 1, + }; + bytes + .get(digits_at) + .is_some_and(u8::is_ascii_digit) + .then_some(digits_at) +} + #[cfg(test)] mod tests { use super::*; @@ -245,4 +272,40 @@ mod tests { let toks = tokenize("-7").unwrap(); assert!(matches!(toks[0].tok, Tok::Int(-7))); } + + fn single_float(src: &str) -> f64 { + let toks = tokenize(src).unwrap(); + assert_eq!(toks.len(), 1, "{src}: {toks:?}"); + match toks[0].tok { + Tok::Float(f) => f, + ref other => panic!("{src}: expected float, got {other:?}"), + } + } + + #[test] + fn tokenize_exponent_floats() { + assert_eq!(single_float("8.983152841195214e-05"), 8.983152841195214e-05); + assert_eq!(single_float("1E3"), 1000.0); + assert_eq!(single_float("-2.5e+2"), -250.0); + assert_eq!(single_float("6e0"), 6.0); + assert_eq!(single_float("-1.5E-3"), -0.0015); + } + + #[test] + fn exponent_without_digits_is_not_consumed() { + // `2e` is an integer then an identifier. + let toks = tokenize("2e").unwrap(); + assert!(matches!(toks[0].tok, Tok::Int(2))); + assert!(matches!(toks[1].tok, Tok::Ident(ref s) if s == "e")); + let toks = tokenize("2e+").unwrap_err(); + assert!(matches!(toks, SqlError::Parse { .. })); + } + + #[test] + fn range_with_exponent_bounds() { + let toks = tokenize("[0..1e3]").unwrap(); + assert!(matches!(toks[1].tok, Tok::Int(0))); + assert!(matches!(toks[2].tok, Tok::DotDot)); + assert!(matches!(toks[3].tok, Tok::Float(f) if f == 1000.0)); + } } diff --git a/nodedb-sql/src/parser/array_stmt/mod.rs b/nodedb-sql/src/parser/array_stmt/mod.rs index 44af28e90..4802795e9 100644 --- a/nodedb-sql/src/parser/array_stmt/mod.rs +++ b/nodedb-sql/src/parser/array_stmt/mod.rs @@ -12,6 +12,7 @@ pub mod lexer; pub mod parse; pub use ast::{ - AlterArrayAst, ArrayStatement, CreateArrayAst, DeleteArrayAst, DropArrayAst, InsertArrayAst, + AlterArrayAction, AlterArrayAst, ArrayStatement, CreateArrayAst, DeleteArrayAst, DropArrayAst, + InsertArrayAst, }; pub use parse::try_parse_array_statement; diff --git a/nodedb-sql/src/parser/array_stmt/parse.rs b/nodedb-sql/src/parser/array_stmt/parse.rs index d51d9ed23..71e529bb5 100644 --- a/nodedb-sql/src/parser/array_stmt/parse.rs +++ b/nodedb-sql/src/parser/array_stmt/parse.rs @@ -275,6 +275,13 @@ mod tests { assert!(try_parse_array_statement(sql).is_err()); } + fn set_options(a: &super::super::ast::AlterArrayAst) -> &[(String, Option)] { + match &a.action { + super::super::ast::AlterArrayAction::SetOptions(set) => set, + other => panic!("expected SET options, got {other:?}"), + } + } + #[test] fn parse_alter_array_single_key() { let sql = "ALTER ARRAY my_array SET (audit_retain_ms = 86400000)"; @@ -282,9 +289,10 @@ mod tests { match stmt { ArrayStatement::Alter(a) => { assert_eq!(a.name, "my_array"); - assert_eq!(a.set.len(), 1); - assert_eq!(a.set[0].0, "audit_retain_ms"); - assert_eq!(a.set[0].1, Some(86_400_000)); + let set = set_options(&a); + assert_eq!(set.len(), 1); + assert_eq!(set[0].0, "audit_retain_ms"); + assert_eq!(set[0].1, Some(86_400_000)); } _ => panic!("expected Alter variant"), } @@ -296,8 +304,9 @@ mod tests { let stmt = try_parse_array_statement(sql).unwrap().unwrap(); match stmt { ArrayStatement::Alter(a) => { - assert_eq!(a.set[0].0, "audit_retain_ms"); - assert_eq!(a.set[0].1, None); + let set = set_options(&a); + assert_eq!(set[0].0, "audit_retain_ms"); + assert_eq!(set[0].1, None); } _ => panic!("expected Alter variant"), } @@ -309,15 +318,14 @@ mod tests { let stmt = try_parse_array_statement(sql).unwrap().unwrap(); match stmt { ArrayStatement::Alter(a) => { - assert_eq!(a.set.len(), 2); + let set = set_options(&a); + assert_eq!(set.len(), 2); assert!( - a.set - .iter() + set.iter() .any(|(k, v)| k == "audit_retain_ms" && *v == Some(5000)) ); assert!( - a.set - .iter() + set.iter() .any(|(k, v)| k == "minimum_audit_retain_ms" && *v == Some(1000)) ); } diff --git a/nodedb-sql/src/parser/array_stmt/parse_impl/alter.rs b/nodedb-sql/src/parser/array_stmt/parse_impl/alter.rs index ef51c59ff..71ae0442b 100644 --- a/nodedb-sql/src/parser/array_stmt/parse_impl/alter.rs +++ b/nodedb-sql/src/parser/array_stmt/parse_impl/alter.rs @@ -3,15 +3,37 @@ //! `ALTER ARRAY` statement parsing. use crate::error::Result; -use crate::parser::array_stmt::ast::AlterArrayAst; +use crate::parser::array_stmt::ast::{AlterArrayAction, AlterArrayAst}; use crate::parser::array_stmt::lexer::Tok; +use crate::types_array::ArrayGeorefAlter; use super::core::Parser; impl<'a> Parser<'a> { pub(in crate::parser::array_stmt::parse) fn parse_alter(&mut self) -> Result { let name = self.expect_ident()?; - self.expect_kw("SET")?; + let action = if self.match_kw("DROP") { + self.expect_kw("GEOREF")?; + AlterArrayAction::Georef(ArrayGeorefAlter::Drop) + } else { + self.expect_kw("SET")?; + if self.match_kw("GEOREF") { + AlterArrayAction::Georef(ArrayGeorefAlter::Set(self.parse_georef()?)) + } else { + AlterArrayAction::SetOptions(self.parse_set_options()?) + } + }; + if !self.at_end() { + return Err(self.err(format!( + "trailing tokens after ALTER ARRAY: {:?}", + self.peek() + ))); + } + Ok(AlterArrayAst { name, action }) + } + + /// Parse `( key = value [, key = value]* )` after `SET`. + fn parse_set_options(&mut self) -> Result)>> { self.expect(&Tok::LParen)?; let mut set = Vec::new(); loop { @@ -69,15 +91,9 @@ impl<'a> Parser<'a> { } } self.expect(&Tok::RParen)?; - if !self.at_end() { - return Err(self.err(format!( - "trailing tokens after ALTER ARRAY: {:?}", - self.peek() - ))); - } if set.is_empty() { return Err(self.err("ALTER ARRAY SET (): at least one key is required")); } - Ok(AlterArrayAst { name, set }) + Ok(set) } } diff --git a/nodedb-sql/src/parser/array_stmt/parse_impl/create.rs b/nodedb-sql/src/parser/array_stmt/parse_impl/create.rs index b9e0cb713..e5f48c94e 100644 --- a/nodedb-sql/src/parser/array_stmt/parse_impl/create.rs +++ b/nodedb-sql/src/parser/array_stmt/parse_impl/create.rs @@ -56,6 +56,11 @@ impl<'a> Parser<'a> { if self.match_kw("TILE_ORDER") { tile_order = self.parse_tile_order()?; } + let georef = if self.match_kw("GEOREF") { + Some(self.parse_georef()?) + } else { + None + }; // Optional `WITH (key = value, ...)` clause. let mut prefix_bits: u8 = 8; @@ -120,6 +125,7 @@ impl<'a> Parser<'a> { tile_extents, cell_order, tile_order, + georef, prefix_bits, audit_retain_ms, minimum_audit_retain_ms, diff --git a/nodedb-sql/src/parser/array_stmt/parse_impl/georef.rs b/nodedb-sql/src/parser/array_stmt/parse_impl/georef.rs new file mode 100644 index 000000000..efdef8fe5 --- /dev/null +++ b/nodedb-sql/src/parser/array_stmt/parse_impl/georef.rs @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! `GEOREF (SRID n, X dim, Y dim [, TRANSFORM (gt0, .., gt5)])` clause parsing. +//! +//! The parser checks the SRID against the supported set and the coefficient +//! count. The planner checks the dims and the coefficient values. + +use nodedb_types::geometry::Srid; + +use crate::error::{Result, SqlError}; +use crate::parser::array_stmt::lexer::Tok; +use crate::types_array::ArrayGeorefAst; + +use super::core::Parser; + +/// Number of coefficients in a GDAL geotransform. +const TRANSFORM_LEN: usize = 6; + +impl<'a> Parser<'a> { + /// Parse the parenthesised body after the `GEOREF` keyword. + pub(super) fn parse_georef(&mut self) -> Result { + self.expect(&Tok::LParen)?; + self.expect_kw("SRID")?; + let code = self.expect_int()?; + let srid = Srid::from_code(code) + .map_err(|e| SqlError::DataException { + detail: format!("GEOREF: {e}"), + })? + .code(); + self.expect(&Tok::Comma)?; + self.expect_kw("X")?; + let x_dim = self.expect_ident()?; + self.expect(&Tok::Comma)?; + self.expect_kw("Y")?; + let y_dim = self.expect_ident()?; + let transform = if self.match_token(&Tok::Comma) { + self.expect_kw("TRANSFORM")?; + Some(self.parse_transform()?) + } else { + None + }; + self.expect(&Tok::RParen)?; + Ok(ArrayGeorefAst { + srid, + x_dim, + y_dim, + transform, + }) + } + + fn parse_transform(&mut self) -> Result<[f64; TRANSFORM_LEN]> { + self.expect(&Tok::LParen)?; + let mut coefficients = Vec::with_capacity(TRANSFORM_LEN); + loop { + coefficients.push(self.expect_float_or_int_as_f64()?); + if !self.match_token(&Tok::Comma) { + break; + } + } + self.expect(&Tok::RParen)?; + let count = coefficients.len(); + <[f64; TRANSFORM_LEN]>::try_from(coefficients).map_err(|_| SqlError::DataException { + detail: format!( + "GEOREF TRANSFORM takes {TRANSFORM_LEN} coefficients in GDAL order, got {count}" + ), + }) + } +} + +#[cfg(test)] +mod tests { + use crate::error::SqlError; + use crate::parser::array_stmt::ast::{AlterArrayAction, ArrayStatement}; + use crate::parser::array_stmt::try_parse_array_statement; + use crate::types_array::{ArrayGeorefAlter, ArrayGeorefAst}; + + const BASE: &str = "CREATE ARRAY scene DIMS (row INT64 [0..1023], col INT64 [0..1023]) \ + ATTRS (b1 UINT16) TILE_EXTENTS (256, 256)"; + + fn create_georef(tail: &str) -> Result, SqlError> { + let sql = format!("{BASE} {tail}"); + match try_parse_array_statement(&sql)? { + Some(ArrayStatement::Create(c)) => Ok(c.georef), + other => panic!("expected CREATE, got {other:?}"), + } + } + + #[test] + fn create_without_georef() { + assert_eq!(create_georef("").unwrap(), None); + } + + #[test] + fn create_world_georef() { + let g = create_georef("GEOREF (SRID 32647, X col, Y row)") + .unwrap() + .unwrap(); + assert_eq!(g.srid, 32647); + assert_eq!(g.x_dim, "col"); + assert_eq!(g.y_dim, "row"); + assert_eq!(g.transform, None); + } + + #[test] + fn create_pixel_georef_with_exponent_floats() { + let g = create_georef( + "TILE_ORDER ROW_MAJOR \ + GEOREF (SRID 4326, X col, Y row, \ + TRANSFORM (100.5, 8.983152841195214e-05, 0, 6.5, 0.0, -8.983152841195214E-05)) \ + WITH (prefix_bits = 4)", + ) + .unwrap() + .unwrap(); + assert_eq!(g.srid, 4326); + assert_eq!( + g.transform, + Some([ + 100.5, + 8.983152841195214e-05, + 0.0, + 6.5, + 0.0, + -8.983152841195214e-05 + ]) + ); + } + + #[test] + fn create_georef_keywords_are_case_insensitive() { + let g = create_georef( + "georef (srid 3375, x col, y row, transform (1e3, 1, 0, -2.5e+2, 0, -1))", + ) + .unwrap() + .unwrap(); + assert_eq!(g.srid, 3375); + assert_eq!(g.transform, Some([1000.0, 1.0, 0.0, -250.0, 0.0, -1.0])); + } + + #[test] + fn unsupported_srid_is_data_exception_naming_supported_codes() { + let err = create_georef("GEOREF (SRID 2154, X col, Y row)").unwrap_err(); + let SqlError::DataException { detail } = err else { + panic!("expected DataException, got {err:?}"); + }; + assert!(detail.contains("2154"), "{detail}"); + for code in ["4326", "32647", "32648", "3375", "3376"] { + assert!(detail.contains(code), "{detail}"); + } + } + + #[test] + fn negative_srid_is_data_exception() { + let err = create_georef("GEOREF (SRID -4326, X col, Y row)").unwrap_err(); + assert!(matches!(err, SqlError::DataException { .. }), "{err:?}"); + } + + #[test] + fn wrong_coefficient_count_is_data_exception() { + for gt in ["(1, 2, 3, 4, 5)", "(1, 2, 3, 4, 5, 6, 7)"] { + let err = create_georef(&format!("GEOREF (SRID 4326, X col, Y row, TRANSFORM {gt})")) + .unwrap_err(); + assert!(matches!(err, SqlError::DataException { .. }), "{err:?}"); + } + } + + #[test] + fn malformed_georef_is_parse_error() { + for tail in [ + "GEOREF (X col, Y row)", + "GEOREF (SRID 4326, Y row, X col)", + "GEOREF (SRID 4326, X col)", + "GEOREF (SRID 4326, X col, Y row", + "GEOREF (SRID 4326, X col, Y row, TRANSFORM (1, 'a', 0, 0, 0, 1))", + "GEOREF (SRID 4326, X col, Y row, ROTATE (1))", + "GEOREF (SRID 4.5, X col, Y row)", + ] { + let err = create_georef(tail).unwrap_err(); + assert!(matches!(err, SqlError::Parse { .. }), "{tail}: {err:?}"); + } + } + + #[test] + fn georef_after_with_is_refused() { + let err = + create_georef("WITH (prefix_bits = 4) GEOREF (SRID 4326, X col, Y row)").unwrap_err(); + assert!(matches!(err, SqlError::Parse { .. }), "{err:?}"); + } + + fn alter(sql: &str) -> Result { + match try_parse_array_statement(sql)? { + Some(ArrayStatement::Alter(a)) => Ok(a.action), + other => panic!("expected ALTER, got {other:?}"), + } + } + + #[test] + fn alter_set_georef() { + let action = alter( + "ALTER ARRAY scene SET GEOREF (SRID 32648, X col, Y row, \ + TRANSFORM (600000, 10, 0, 700000, 0, -10))", + ) + .unwrap(); + let AlterArrayAction::Georef(ArrayGeorefAlter::Set(g)) = action else { + panic!("expected SET GEOREF, got {action:?}"); + }; + assert_eq!(g.srid, 32648); + assert_eq!( + g.transform, + Some([600_000.0, 10.0, 0.0, 700_000.0, 0.0, -10.0]) + ); + } + + #[test] + fn alter_drop_georef() { + let action = alter("ALTER ARRAY scene DROP GEOREF").unwrap(); + assert_eq!(action, AlterArrayAction::Georef(ArrayGeorefAlter::Drop)); + } + + #[test] + fn alter_georef_errors() { + let err = alter("ALTER ARRAY scene SET GEOREF (SRID 27700, X col, Y row)").unwrap_err(); + assert!(matches!(err, SqlError::DataException { .. }), "{err:?}"); + for sql in [ + "ALTER ARRAY scene DROP GEOREF extra", + "ALTER ARRAY scene DROP", + "ALTER ARRAY scene DROP COLUMN x", + "ALTER ARRAY scene SET GEOREF (SRID 4326, X col, Y row) extra", + ] { + let err = alter(sql).unwrap_err(); + assert!(matches!(err, SqlError::Parse { .. }), "{sql}: {err:?}"); + } + } +} diff --git a/nodedb-sql/src/parser/array_stmt/parse_impl/mod.rs b/nodedb-sql/src/parser/array_stmt/parse_impl/mod.rs index 1d9a22f51..7769d845f 100644 --- a/nodedb-sql/src/parser/array_stmt/parse_impl/mod.rs +++ b/nodedb-sql/src/parser/array_stmt/parse_impl/mod.rs @@ -7,6 +7,7 @@ mod core; mod create; mod delete_stmt; mod drop; +mod georef; mod insert; pub(in crate::parser::array_stmt::parse) use core::Parser; diff --git a/nodedb-sql/src/planner/array_ddl.rs b/nodedb-sql/src/planner/array_ddl.rs index ba06f0240..fff2d5cfd 100644 --- a/nodedb-sql/src/planner/array_ddl.rs +++ b/nodedb-sql/src/planner/array_ddl.rs @@ -11,10 +11,12 @@ use crate::types::{AlterArrayPlan, CreateArrayPlan}; use std::collections::HashSet; +use crate::catalog::SqlCatalog; use crate::error::{Result, SqlError}; -use crate::parser::array_stmt::{AlterArrayAst, CreateArrayAst, DropArrayAst}; +use crate::parser::array_stmt::{AlterArrayAction, AlterArrayAst, CreateArrayAst, DropArrayAst}; +use crate::planner::array_georef::check_georef; use crate::types::SqlPlan; -use crate::types_array::{ArrayDimType, ArrayDomainBound}; +use crate::types_array::{ArrayDimType, ArrayDomainBound, ArrayGeorefAlter}; pub fn plan_create_array(ast: &CreateArrayAst) -> Result { if ast.name.is_empty() { @@ -103,6 +105,9 @@ pub fn plan_create_array(ast: &CreateArrayAst) -> Result { ), }); } + if let Some(georef) = &ast.georef { + check_georef(&ast.name, georef, &ast.dims)?; + } Ok(SqlPlan::CreateArray(CreateArrayPlan { name: ast.name.clone(), dims: ast.dims.clone(), @@ -110,47 +115,88 @@ pub fn plan_create_array(ast: &CreateArrayAst) -> Result { tile_extents: ast.tile_extents.clone(), cell_order: ast.cell_order, tile_order: ast.tile_order, + georef: ast.georef.clone(), prefix_bits: ast.prefix_bits, audit_retain_ms: ast.audit_retain_ms, minimum_audit_retain_ms: ast.minimum_audit_retain_ms, })) } -pub fn plan_alter_array(ast: &AlterArrayAst) -> Result { +pub fn plan_alter_array(ast: &AlterArrayAst, catalog: &dyn SqlCatalog) -> Result { if ast.name.is_empty() { return Err(SqlError::Parse { detail: "ALTER ARRAY: array name must not be empty".into(), }); } - if ast.set.is_empty() { + match &ast.action { + AlterArrayAction::SetOptions(set) => plan_alter_options(&ast.name, set), + AlterArrayAction::Georef(change) => plan_alter_georef(&ast.name, change, catalog), + } +} + +/// `ALTER ARRAY SET GEOREF (...)` or `DROP GEOREF`, checked against +/// the array's current dims and georef. +fn plan_alter_georef( + name: &str, + change: &ArrayGeorefAlter, + catalog: &dyn SqlCatalog, +) -> Result { + let view = catalog + .lookup_array(name) + .ok_or_else(|| SqlError::UnknownTable { + name: name.to_string(), + })?; + match change { + ArrayGeorefAlter::Set(georef) => check_georef(name, georef, &view.dims)?, + ArrayGeorefAlter::Drop => { + if view.georef.is_none() { + return Err(SqlError::ObjectNotInPrerequisiteState { + object: format!("array '{name}'"), + detail: format!("ALTER ARRAY {name} DROP GEOREF: the array has no GEOREF"), + }); + } + } + } + Ok(SqlPlan::AlterArray(AlterArrayPlan { + name: name.to_string(), + audit_retain_ms: None, + minimum_audit_retain_ms: None, + georef: Some(change.clone()), + })) +} + +/// `ALTER ARRAY SET (key = value, ...)`. +fn plan_alter_options(name: &str, set: &[(String, Option)]) -> Result { + if set.is_empty() { return Err(SqlError::Parse { - detail: format!("ALTER ARRAY {}: SET clause is empty", ast.name), + detail: format!("ALTER ARRAY {name}: SET clause is empty"), }); } let mut audit_retain_ms: Option> = None; let mut minimum_audit_retain_ms: Option = None; - for (key, value) in &ast.set { + for (key, value) in set { + if value.is_some_and(|v| v < 0) { + return Err(SqlError::Parse { + detail: format!("ALTER ARRAY {name}: {key} must not be negative"), + }); + } match key.as_str() { "audit_retain_ms" => { audit_retain_ms = Some(*value); } "minimum_audit_retain_ms" => { let n = value.ok_or_else(|| SqlError::Parse { - detail: format!( - "ALTER ARRAY {}: minimum_audit_retain_ms cannot be NULL", - ast.name - ), + detail: format!("ALTER ARRAY {name}: minimum_audit_retain_ms cannot be NULL"), })?; minimum_audit_retain_ms = Some(n as u64); } other => { return Err(SqlError::Parse { detail: format!( - "ALTER ARRAY {}: unknown SET key `{other}`; \ - expected `audit_retain_ms` or `minimum_audit_retain_ms`", - ast.name + "ALTER ARRAY {name}: unknown SET key `{other}`; \ + expected `audit_retain_ms` or `minimum_audit_retain_ms`" ), }); } @@ -158,9 +204,10 @@ pub fn plan_alter_array(ast: &AlterArrayAst) -> Result { } Ok(SqlPlan::AlterArray(AlterArrayPlan { - name: ast.name.clone(), + name: name.to_string(), audit_retain_ms, minimum_audit_retain_ms, + georef: None, })) } @@ -216,7 +263,8 @@ fn domain_lo_le_hi(lo: &ArrayDomainBound, hi: &ArrayDomainBound) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::types_array::{ArrayAttrAst, ArrayAttrType, ArrayDimAst}; + use crate::catalog::ArrayCatalogView; + use crate::types_array::{ArrayAttrAst, ArrayAttrType, ArrayDimAst, ArrayGeorefAst}; fn ok_ast() -> CreateArrayAst { CreateArrayAst { @@ -235,12 +283,195 @@ mod tests { tile_extents: vec![4], cell_order: Default::default(), tile_order: Default::default(), + georef: None, prefix_bits: 8, audit_retain_ms: None, minimum_audit_retain_ms: None, } } + fn int_dim(name: &str) -> ArrayDimAst { + ArrayDimAst { + name: name.into(), + dtype: ArrayDimType::Int64, + lo: ArrayDomainBound::Int64(0), + hi: ArrayDomainBound::Int64(1023), + } + } + + fn georef(x: &str, y: &str, transform: Option<[f64; 6]>) -> ArrayGeorefAst { + ArrayGeorefAst { + srid: 32647, + x_dim: x.into(), + y_dim: y.into(), + transform, + } + } + + const GT: [f64; 6] = [600_000.0, 10.0, 0.0, 700_000.0, 0.0, -10.0]; + + fn raster_ast(georef: Option) -> CreateArrayAst { + CreateArrayAst { + dims: vec![int_dim("row"), int_dim("col")], + tile_extents: vec![256, 256], + georef, + ..ok_ast() + } + } + + #[test] + fn create_carries_georef() { + let g = georef("col", "row", Some(GT)); + let SqlPlan::CreateArray(plan) = plan_create_array(&raster_ast(Some(g.clone()))).unwrap() + else { + panic!("expected CreateArray"); + }; + assert_eq!(plan.georef, Some(g)); + } + + #[test] + fn create_refuses_bad_georef_with_sqlstate_class() { + let err = plan_create_array(&raster_ast(Some(georef("col", "lat", None)))).unwrap_err(); + assert!(matches!(err, SqlError::UnknownColumn { .. }), "{err:?}"); + let mut srid = georef("col", "row", None); + srid.srid = 2154; + let err = plan_create_array(&raster_ast(Some(srid))).unwrap_err(); + assert!(matches!(err, SqlError::DataException { .. }), "{err:?}"); + let singular = georef("col", "row", Some([0.0, 1.0, 1.0, 0.0, 1.0, 1.0])); + let err = plan_create_array(&raster_ast(Some(singular))).unwrap_err(); + assert!(matches!(err, SqlError::DataException { .. }), "{err:?}"); + } + + struct Catalog(Option); + + impl SqlCatalog for Catalog { + fn get_collection( + &self, + _: nodedb_types::DatabaseId, + _: &str, + ) -> std::result::Result< + Option, + crate::catalog::SqlCatalogError, + > { + Ok(None) + } + fn lookup_array(&self, _: &str) -> Option { + self.0.clone() + } + } + + fn view(georef: Option) -> ArrayCatalogView { + ArrayCatalogView { + name: "scene".into(), + dims: vec![int_dim("row"), int_dim("col")], + attrs: ok_ast().attrs, + tile_extents: vec![256, 256], + georef, + } + } + + fn alter(action: AlterArrayAction, catalog: &Catalog) -> Result { + plan_alter_array( + &AlterArrayAst { + name: "scene".into(), + action, + }, + catalog, + ) + } + + #[test] + fn alter_set_georef_plans_the_change() { + let g = georef("col", "row", Some(GT)); + let change = ArrayGeorefAlter::Set(g); + let SqlPlan::AlterArray(plan) = alter( + AlterArrayAction::Georef(change.clone()), + &Catalog(Some(view(None))), + ) + .unwrap() else { + panic!("expected AlterArray"); + }; + assert_eq!(plan.georef, Some(change)); + assert_eq!(plan.audit_retain_ms, None); + assert_eq!(plan.minimum_audit_retain_ms, None); + } + + #[test] + fn alter_set_georef_refusals() { + let catalog = Catalog(Some(view(None))); + let unknown = ArrayGeorefAlter::Set(georef("col", "lat", None)); + let err = alter(AlterArrayAction::Georef(unknown), &catalog).unwrap_err(); + assert!(matches!(err, SqlError::UnknownColumn { .. }), "{err:?}"); + let mut inf = GT; + inf[5] = f64::NEG_INFINITY; + let bad = ArrayGeorefAlter::Set(georef("col", "row", Some(inf))); + let err = alter(AlterArrayAction::Georef(bad), &catalog).unwrap_err(); + assert!(matches!(err, SqlError::DataException { .. }), "{err:?}"); + let missing = alter( + AlterArrayAction::Georef(ArrayGeorefAlter::Drop), + &Catalog(None), + ) + .unwrap_err(); + assert!( + matches!(missing, SqlError::UnknownTable { .. }), + "{missing:?}" + ); + } + + #[test] + fn alter_drop_georef_needs_a_georef() { + let err = alter( + AlterArrayAction::Georef(ArrayGeorefAlter::Drop), + &Catalog(Some(view(None))), + ) + .unwrap_err(); + assert!( + matches!(err, SqlError::ObjectNotInPrerequisiteState { .. }), + "{err:?}" + ); + let plan = alter( + AlterArrayAction::Georef(ArrayGeorefAlter::Drop), + &Catalog(Some(view(Some(georef("col", "row", None))))), + ) + .unwrap(); + assert!(matches!( + plan, + SqlPlan::AlterArray(AlterArrayPlan { + georef: Some(ArrayGeorefAlter::Drop), + .. + }) + )); + } + + #[test] + fn alter_options_keep_georef_unchanged() { + let plan = alter( + AlterArrayAction::SetOptions(vec![("audit_retain_ms".into(), Some(5))]), + &Catalog(None), + ) + .unwrap(); + assert!(matches!( + plan, + SqlPlan::AlterArray(AlterArrayPlan { + georef: None, + audit_retain_ms: Some(Some(5)), + .. + }) + )); + } + + #[test] + fn alter_options_refuse_negative_values() { + for key in ["audit_retain_ms", "minimum_audit_retain_ms"] { + let err = alter( + AlterArrayAction::SetOptions(vec![(key.into(), Some(-1))]), + &Catalog(None), + ) + .expect_err("negative refused"); + assert!(matches!(err, SqlError::Parse { .. }), "{key}: {err:?}"); + } + } + #[test] fn happy_path() { let plan = plan_create_array(&ok_ast()).unwrap(); diff --git a/nodedb-sql/src/planner/array_dml.rs b/nodedb-sql/src/planner/array_dml.rs index df6547d4d..c67e4e638 100644 --- a/nodedb-sql/src/planner/array_dml.rs +++ b/nodedb-sql/src/planner/array_dml.rs @@ -195,6 +195,7 @@ mod tests { nullable: true, }], tile_extents: vec![1, 1_000_000], + georef: None, } } diff --git a/nodedb-sql/src/planner/array_fn/maint_fn.rs b/nodedb-sql/src/planner/array_fn/maint_fn.rs index e204a02c3..195292ac5 100644 --- a/nodedb-sql/src/planner/array_fn/maint_fn.rs +++ b/nodedb-sql/src/planner/array_fn/maint_fn.rs @@ -109,6 +109,7 @@ mod tests { }, ], tile_extents: vec![1, 1_000_000], + georef: None, } } diff --git a/nodedb-sql/src/planner/array_fn/pushdown.rs b/nodedb-sql/src/planner/array_fn/pushdown.rs index 1ceb437da..b4432f1cd 100644 --- a/nodedb-sql/src/planner/array_fn/pushdown.rs +++ b/nodedb-sql/src/planner/array_fn/pushdown.rs @@ -342,6 +342,7 @@ mod tests { nullable: true, }], tile_extents: vec![2, 2], + georef: None, } } @@ -351,6 +352,7 @@ mod tests { slice: ArraySliceAst::default(), attr_projection: Vec::new(), limit, + zone: None, temporal: TemporalScope::default(), } } diff --git a/nodedb-sql/src/planner/array_fn/table_fn.rs b/nodedb-sql/src/planner/array_fn/table_fn.rs index d8ca61c55..776948e97 100644 --- a/nodedb-sql/src/planner/array_fn/table_fn.rs +++ b/nodedb-sql/src/planner/array_fn/table_fn.rs @@ -10,7 +10,7 @@ use super::helpers::{ collect_args, expect_string_array, expect_string_literal, expect_u32, is_null_literal, parse_slice_arg, require_array_name, }; -use super::zone::parse_zone_args; +use super::zone::parse_optional_zone; use crate::error::{Result, SqlError}; use crate::resolver::array_tvf::check_agg_reducer; use crate::temporal::TemporalScope; @@ -115,14 +115,20 @@ fn plan_slice( catalog: &dyn SqlCatalog, temporal: TemporalScope, ) -> Result { - if args.len() < 2 || args.len() > 4 { + if !(2..=6).contains(&args.len()) { return Err(SqlError::Unsupported { detail: format!( - "ARRAY_SLICE expects 2..=4 args (name, slice_obj, [attrs], [limit]); got {}", + "ARRAY_SLICE expects 2..=6 args (name, slice_obj, [attrs], [limit], \ + [zone_wkt, zone_dims]); got {}", args.len() ), }); } + if args.len() == 5 { + return Err(SqlError::Unsupported { + detail: "ARRAY_SLICE: zone_wkt needs zone_dims; pass both, or neither".into(), + }); + } let name = require_array_name(args, 0, "ARRAY_SLICE", catalog)?; let view = catalog .lookup_array(&name) @@ -132,15 +138,13 @@ fn plan_slice( let slice = parse_slice_arg(&args[1], "ARRAY_SLICE", &name, &view)?; - let attr_projection = if args.len() >= 3 { - match &args[2] { - ast::Expr::Value(v) if matches!(v.value, ast::Value::SingleQuotedString(ref s) if s == "*") => { - Vec::new() - } - _ => expect_string_array(&args[2], "ARRAY_SLICE attr projection")?, + let attr_projection = match args.get(2) { + None => Vec::new(), + Some(arg) if is_null_literal(arg) => Vec::new(), + Some(ast::Expr::Value(v)) if matches!(v.value, ast::Value::SingleQuotedString(ref s) if s == "*") => { + Vec::new() } - } else { - Vec::new() + Some(arg) => expect_string_array(arg, "ARRAY_SLICE attr projection")?, }; // Validate attr names against the catalog. for attr in &attr_projection { @@ -151,17 +155,18 @@ fn plan_slice( } } - let limit = if args.len() >= 4 { - expect_u32(&args[3], "ARRAY_SLICE limit")? - } else { - 0 + let limit = match args.get(3) { + Some(arg) if !is_null_literal(arg) => expect_u32(arg, "ARRAY_SLICE limit")?, + _ => 0, }; + let zone = parse_optional_zone("ARRAY_SLICE", args, 4, &name, &view)?; Ok(ArraySlicePlan { name, slice, attr_projection, limit, + zone, temporal, }) } @@ -203,6 +208,7 @@ fn plan_project( slice: ArraySliceAst::default(), attr_projection, limit: 0, + zone: None, temporal, }) } @@ -267,10 +273,7 @@ fn plan_agg( _ => ArraySliceAst::default(), }; - let zone = match (args.get(5), args.get(6)) { - (Some(wkt), Some(dims)) => Some(parse_zone_args(wkt, dims, &name, &view)?), - _ => None, - }; + let zone = parse_optional_zone("ARRAY_AGG", args, 5, &name, &view)?; Ok(ArrayAggPlan { name, @@ -315,6 +318,16 @@ fn plan_elementwise( ), }); } + // The schema hash excludes the georef, so this is the only check that + // the two arrays place their cells in the same world frame. + if lview.georef != rview.georef { + return Err(SqlError::DataException { + detail: format!( + "ARRAY_ELEMENTWISE: arrays '{left}' and '{right}' must have equal GEOREF \ + (SRID, X and Y dims, and TRANSFORM)" + ), + }); + } let op_str = expect_string_literal(&args[2], "ARRAY_ELEMENTWISE op")?; let op = ArrayBinaryOpAst::parse(&op_str).ok_or_else(|| SqlError::Unsupported { detail: format!("ARRAY_ELEMENTWISE: unknown op '{op_str}' (want add/sub/mul/div)"), @@ -343,7 +356,8 @@ mod tests { use crate::types::{ArrayAggPlan, ArrayElementwisePlan, ArraySlicePlan}; use crate::types::{CollectionInfo, SqlCatalog, SqlPlan}; use crate::types_array::{ - ArrayAttrAst, ArrayAttrType, ArrayDimAst, ArrayDimType, ArrayDomainBound, ArrayReducerAst, + ArrayAttrAst, ArrayAttrType, ArrayDimAst, ArrayDimType, ArrayDomainBound, ArrayGeorefAst, + ArrayReducerAst, }; struct StubCatalog { @@ -399,6 +413,7 @@ mod tests { }, ], tile_extents: vec![1, 1_000_000], + georef: None, } } @@ -719,4 +734,62 @@ mod tests { plan_one("SELECT * FROM ARRAY_ELEMENTWISE('left', 'right', 'wat', 'qual')").is_err() ); } + + fn georef(srid: u32, transform: Option<[f64; 6]>) -> Option { + Some(ArrayGeorefAst { + srid, + x_dim: "pos".into(), + y_dim: "chrom".into(), + transform, + }) + } + + fn elementwise_with( + left: Option, + right: Option, + ) -> Result { + let catalog = StubCatalog { + view: Some(ArrayCatalogView { + georef: left, + ..view() + }), + right_view: Some(ArrayCatalogView { + georef: right, + ..view() + }), + }; + let stmts = parse_sql("SELECT * FROM ARRAY_ELEMENTWISE('left', 'right', 'add', 'qual')")?; + let sqlparser::ast::Statement::Query(q) = &stmts[0] else { + panic!("not a query"); + }; + crate::planner::select::plan_query( + q, + &catalog, + &FunctionRegistry::new(), + crate::TemporalScope::default(), + ) + } + + const GT: [f64; 6] = [600_000.0, 10.0, 0.0, 700_000.0, 0.0, -10.0]; + + #[test] + fn elementwise_equal_georefs_pass() { + assert!(elementwise_with(georef(32647, Some(GT)), georef(32647, Some(GT))).is_ok()); + assert!(elementwise_with(georef(4326, None), georef(4326, None)).is_ok()); + } + + #[test] + fn elementwise_different_georefs_refused() { + let mut shifted = GT; + shifted[0] += 10.0; + for (left, right) in [ + (georef(32647, Some(GT)), georef(32647, Some(shifted))), + (georef(32647, Some(GT)), georef(32648, Some(GT))), + (georef(32647, Some(GT)), None), + (None, georef(4326, None)), + ] { + let err = elementwise_with(left, right).unwrap_err(); + assert!(matches!(err, SqlError::DataException { .. }), "{err:?}"); + } + } } diff --git a/nodedb-sql/src/planner/array_fn/zone.rs b/nodedb-sql/src/planner/array_fn/zone.rs index 311fa1270..5750b80e7 100644 --- a/nodedb-sql/src/planner/array_fn/zone.rs +++ b/nodedb-sql/src/planner/array_fn/zone.rs @@ -1,83 +1,399 @@ // SPDX-License-Identifier: Apache-2.0 -//! The polygon zone argument pair of `ARRAY_AGG`: a WKT `POLYGON` or -//! `MULTIPOLYGON` and the two numeric dims it is drawn over, x then y. +//! The polygon zone argument pair of `ARRAY_AGG` and `ARRAY_SLICE`: an +//! (E)WKT `POLYGON` or `MULTIPOLYGON` and the two numeric dims it is drawn +//! over, x then y. +//! +//! - No `SRID=` prefix: the positions are dim coordinates, and `zone_dims` +//! names the two dims. +//! - `SRID=;` prefix: the positions are in SRID `n`. The array needs a +//! GEOREF (`55000` otherwise). `zone_dims` NULL means the GEOREF axes. A +//! non-NULL `zone_dims` must name exactly those axes (`55000` otherwise). use sqlparser::ast; -use nodedb_types::geometry::Geometry; +use nodedb_types::geometry::{Geometry, Srid}; -use super::helpers::{expect_string_array, expect_string_literal}; +use super::helpers::{expect_string_array, expect_string_literal, is_null_literal}; use crate::catalog::ArrayCatalogView; use crate::error::{Result, SqlError}; use crate::types_array::{ArrayDimType, ArrayZoneAst}; -/// Parse `zone_wkt` and `zone_dims` of an `ARRAY_AGG` over array `name`. -pub(super) fn parse_zone_args( +/// The zone of `func` from `args[at]` (`zone_wkt`) and `args[at + 1]` +/// (`zone_dims`). Absent or NULL `zone_wkt` means no zone. +pub(super) fn parse_optional_zone( + func: &str, + args: &[ast::Expr], + at: usize, + name: &str, + view: &ArrayCatalogView, +) -> Result> { + match (args.get(at), args.get(at + 1)) { + (Some(wkt), Some(dims)) if !is_null_literal(wkt) => { + parse_zone_args(func, wkt, dims, name, view).map(Some) + } + (Some(_), Some(dims)) if !is_null_literal(dims) => Err(SqlError::Unsupported { + detail: format!("{func}: zone_dims needs a zone_wkt; pass both, or neither"), + }), + _ => Ok(None), + } +} + +/// Parse the `zone_wkt` and `zone_dims` arguments of `func` over array `name`. +fn parse_zone_args( + func: &str, wkt_expr: &ast::Expr, dims_expr: &ast::Expr, name: &str, view: &ArrayCatalogView, ) -> Result { - let wkt = expect_string_literal(wkt_expr, "ARRAY_AGG zone")?; - let geometry = - nodedb_spatial::geometry_from_wkt(&wkt).ok_or_else(|| SqlError::InvalidFunction { - detail: format!("ARRAY_AGG: zone is not valid WKT: {wkt}"), + let wkt = expect_string_literal(wkt_expr, &format!("{func} zone"))?; + let mut ewkt = + nodedb_spatial::geometry_from_ewkt(&wkt).ok_or_else(|| SqlError::InvalidFunction { + detail: format!("{func}: zone is not valid WKT or EWKT: {wkt}"), })?; - let issues = nodedb_spatial::validate::validate_geometry(&geometry); + // Membership is an even-odd test, so ring direction never changes it. + // Orienting first accepts a zone drawn clockwise. + nodedb_spatial::orient_rings(&mut ewkt.geometry); + let issues = nodedb_spatial::validate::validate_geometry(&ewkt.geometry); if !issues.is_empty() { return Err(SqlError::InvalidFunction { - detail: format!("ARRAY_AGG: invalid zone: {}", issues.join("; ")), + detail: format!("{func}: invalid zone: {}", issues.join("; ")), }); } - let polygons = match geometry { + let polygons = match ewkt.geometry { Geometry::Polygon { coordinates } => vec![coordinates], Geometry::MultiPolygon { coordinates } => coordinates, other => { return Err(SqlError::InvalidFunction { detail: format!( - "ARRAY_AGG: zone must be a POLYGON or MULTIPOLYGON, got {}", + "{func}: zone must be a POLYGON or MULTIPOLYGON, got {}", other.geometry_type() ), }); } }; + let dims = if is_null_literal(dims_expr) { + None + } else { + Some(expect_string_array( + dims_expr, + &format!("{func} zone dims"), + )?) + }; + let (x_dim, y_dim, srid) = match ewkt.srid { + None => { + let dims = dims.ok_or_else(|| SqlError::Unsupported { + detail: format!( + "{func}: a zone without an SRID prefix needs zone_dims; name two dims, \ + x then y" + ), + })?; + let (x_dim, y_dim) = two_dims(func, dims)?; + (x_dim, y_dim, None) + } + Some(code) => { + let srid = Srid::from_code(code).map_err(|e| SqlError::DataException { + detail: format!("{func}: zone SRID: {e}"), + })?; + let (x_dim, y_dim) = georef_axes(func, name, view, dims)?; + (x_dim, y_dim, Some(srid.code())) + } + }; + for dim in [&x_dim, &y_dim] { + check_zone_dim(func, name, view, dim)?; + } + Ok(ArrayZoneAst { + polygons, + x_dim, + y_dim, + srid, + }) +} - let dims = expect_string_array(dims_expr, "ARRAY_AGG zone dims")?; +/// The `[x, y]` pair `zone_dims` names. +fn two_dims(func: &str, dims: Vec) -> Result<(String, String)> { let [x_dim, y_dim] = <[String; 2]>::try_from(dims).map_err(|dims| SqlError::Unsupported { detail: format!( - "ARRAY_AGG: zone dims must name two dims, x then y; got {}", + "{func}: zone dims must name two dims, x then y; got {}", dims.len() ), })?; if x_dim == y_dim { return Err(SqlError::Unsupported { - detail: format!("ARRAY_AGG: zone dims must be two different dims; got '{x_dim}' twice"), + detail: format!("{func}: zone dims must be two different dims; got '{x_dim}' twice"), }); } - for dim in [&x_dim, &y_dim] { - let spec = - view.dims - .iter() - .find(|d| &d.name == dim) - .ok_or_else(|| SqlError::Unsupported { - detail: format!("ARRAY_AGG: array '{name}' has no dim '{dim}'"), - })?; - match spec.dtype { - ArrayDimType::Int64 | ArrayDimType::Float64 => {} - ArrayDimType::TimestampMs | ArrayDimType::String => { - return Err(SqlError::TypeMismatch { - detail: format!( - "ARRAY_AGG: zone dim '{dim}' is {:?}; a zone needs an INT64 or FLOAT64 dim", - spec.dtype - ), - }); - } + Ok((x_dim, y_dim)) +} + +/// The GEOREF axes of array `name`, checked against `dims` when given. +fn georef_axes( + func: &str, + name: &str, + view: &ArrayCatalogView, + dims: Option>, +) -> Result<(String, String)> { + let georef = view + .georef + .as_ref() + .ok_or_else(|| SqlError::ObjectNotInPrerequisiteState { + object: format!("array '{name}'"), + detail: format!( + "{func}: array '{name}' has no GEOREF; a zone with an SRID prefix needs one" + ), + })?; + let axes = (georef.x_dim.clone(), georef.y_dim.clone()); + if let Some(dims) = dims + && dims != [axes.0.as_str(), axes.1.as_str()] + { + return Err(SqlError::ObjectNotInPrerequisiteState { + object: format!("array '{name}'"), + detail: format!( + "{func}: a zone with an SRID prefix is drawn over the GEOREF axes ['{}', '{}'] \ + of array '{name}'; zone_dims {dims:?} names others. Pass NULL or those axes", + axes.0, axes.1 + ), + }); + } + Ok(axes) +} + +/// `dim` exists on array `name` and is numeric. +fn check_zone_dim(func: &str, name: &str, view: &ArrayCatalogView, dim: &str) -> Result<()> { + let spec = view + .dims + .iter() + .find(|d| d.name == dim) + .ok_or_else(|| SqlError::Unsupported { + detail: format!("{func}: array '{name}' has no dim '{dim}'"), + })?; + match spec.dtype { + ArrayDimType::Int64 | ArrayDimType::Float64 => Ok(()), + ArrayDimType::TimestampMs | ArrayDimType::String => Err(SqlError::TypeMismatch { + detail: format!( + "{func}: zone dim '{dim}' is {:?}; a zone needs an INT64 or FLOAT64 dim", + spec.dtype + ), + }), + } +} + +#[cfg(test)] +mod tests { + use crate::catalog::{ArrayCatalogView, SqlCatalogError}; + use crate::error::{Result, SqlError}; + use crate::functions::registry::FunctionRegistry; + use crate::parser::statement::parse_sql; + use crate::types::*; + use crate::types_array::{ + ArrayAttrAst, ArrayAttrType, ArrayDimAst, ArrayDimType, ArrayDomainBound, ArrayGeorefAst, + ArrayZoneAst, + }; + + /// `img` carries a pixel GEOREF over (col, row). `plain` has none. + struct ZoneCatalog; + + impl SqlCatalog for ZoneCatalog { + fn get_collection( + &self, + _: nodedb_types::DatabaseId, + _: &str, + ) -> std::result::Result, SqlCatalogError> { + Ok(None) + } + + fn lookup_array(&self, name: &str) -> Option { + let dim = |name: &str| ArrayDimAst { + name: name.into(), + dtype: ArrayDimType::Int64, + lo: ArrayDomainBound::Int64(0), + hi: ArrayDomainBound::Int64(255), + }; + let georef = match name { + "img" => Some(ArrayGeorefAst { + srid: 32647, + x_dim: "col".into(), + y_dim: "row".into(), + transform: Some([600_000.0, 30.0, 0.0, 700_000.0, 0.0, -30.0]), + }), + "plain" => None, + _ => return None, + }; + Some(ArrayCatalogView { + name: name.into(), + dims: vec![dim("row"), dim("col")], + attrs: vec![ArrayAttrAst { + name: "b1".into(), + dtype: ArrayAttrType::Float64, + nullable: true, + }], + tile_extents: vec![64, 64], + georef, + }) } } - Ok(ArrayZoneAst { - polygons, - x_dim, - y_dim, - }) + + const ZONE: &str = "'SRID=4326;POLYGON((99 6, 100 6, 100 6.5, 99 6.5, 99 6))'"; + + fn plan(sql: &str) -> Result { + let stmts = parse_sql(sql)?; + let sqlparser::ast::Statement::Query(q) = &stmts[0] else { + panic!("not a query: {sql}"); + }; + crate::planner::select::plan_query( + q, + &ZoneCatalog, + &FunctionRegistry::new(), + crate::TemporalScope::default(), + ) + } + + fn agg_zone(sql: &str) -> ArrayZoneAst { + match plan(sql).unwrap_or_else(|e| panic!("{sql}: {e}")) { + SqlPlan::ArrayAgg(ArrayAggPlan { zone, .. }) => zone.expect("zone"), + other => panic!("expected ArrayAgg, got {other:?}"), + } + } + + fn slice_zone(sql: &str) -> Option { + match plan(sql).unwrap_or_else(|e| panic!("{sql}: {e}")) { + SqlPlan::Cte(CtePlan { + mut definitions, .. + }) => match definitions.remove(0).1 { + SqlPlan::ArraySlice(ArraySlicePlan { zone, .. }) => zone, + other => panic!("expected ArraySlice, got {other:?}"), + }, + other => panic!("expected Cte, got {other:?}"), + } + } + + #[test] + fn a_null_zone_dims_takes_the_georef_axes() { + let zone = agg_zone(&format!( + "SELECT * FROM ARRAY_AGG('img', 'b1', 'sum', NULL, NULL, {ZONE}, NULL)" + )); + assert_eq!((zone.x_dim.as_str(), zone.y_dim.as_str()), ("col", "row")); + assert_eq!(zone.srid, Some(4326)); + assert_eq!(zone.polygons[0][0].len(), 5); + } + + #[test] + fn a_clockwise_zone_plans_with_its_rings_oriented() { + let zone = agg_zone( + "SELECT * FROM ARRAY_AGG('img', 'b1', 'sum', NULL, NULL, \ + 'SRID=4326;POLYGON((99 6, 99 6.5, 100 6.5, 100 6, 99 6))', NULL)", + ); + let ccw = agg_zone(&format!( + "SELECT * FROM ARRAY_AGG('img', 'b1', 'sum', NULL, NULL, {ZONE}, NULL)" + )); + let mut planned = zone.polygons[0][0].clone(); + assert!( + nodedb_spatial::validate::validate_geometry( + &nodedb_types::geometry::Geometry::polygon(vec![planned.clone()]) + ) + .is_empty(), + "{planned:?}" + ); + // Same vertex set as the counter-clockwise spelling. + let mut expected = ccw.polygons[0][0].clone(); + let key = |p: &[f64; 2]| (p[0].to_bits(), p[1].to_bits()); + planned.sort_by_key(key); + expected.sort_by_key(key); + assert_eq!(planned, expected); + } + + #[test] + fn zone_dims_equal_to_the_georef_axes_plan() { + let zone = agg_zone(&format!( + "SELECT * FROM ARRAY_AGG('img', 'b1', 'sum', NULL, NULL, {ZONE}, ARRAY['col', 'row'])" + )); + assert_eq!(zone.srid, Some(4326)); + } + + #[test] + fn zone_dims_other_than_the_georef_axes_are_55000() { + let err = plan(&format!( + "SELECT * FROM ARRAY_AGG('img', 'b1', 'sum', NULL, NULL, {ZONE}, ARRAY['row', 'col'])" + )) + .expect_err("refused"); + assert!( + matches!(err, SqlError::ObjectNotInPrerequisiteState { .. }), + "{err:?}" + ); + } + + #[test] + fn an_srid_zone_without_georef_is_55000() { + for sql in [ + format!("SELECT * FROM ARRAY_AGG('plain', 'b1', 'sum', NULL, NULL, {ZONE}, NULL)"), + format!("SELECT * FROM ARRAY_SLICE('plain', '{{row: [0, 9]}}', '*', 0, {ZONE}, NULL)"), + ] { + let err = plan(&sql).expect_err("refused"); + assert!( + matches!(err, SqlError::ObjectNotInPrerequisiteState { .. }), + "{sql}: {err:?}" + ); + } + } + + #[test] + fn an_unsupported_zone_srid_is_a_data_exception() { + let err = plan( + "SELECT * FROM ARRAY_AGG('img', 'b1', 'sum', NULL, NULL, \ + 'SRID=2154;POLYGON((0 0, 1 0, 1 1, 0 0))', NULL)", + ) + .expect_err("refused"); + assert!(matches!(err, SqlError::DataException { .. }), "{err:?}"); + } + + #[test] + fn a_zone_without_srid_still_needs_its_dims() { + let err = plan( + "SELECT * FROM ARRAY_AGG('img', 'b1', 'sum', NULL, NULL, \ + 'POLYGON((0 0, 1 0, 1 1, 0 0))', NULL)", + ) + .expect_err("refused"); + assert!(format!("{err}").contains("zone_dims"), "{err}"); + } + + #[test] + fn slice_takes_two_to_six_args_and_refuses_five() { + assert!(slice_zone("SELECT * FROM ARRAY_SLICE('img', '{row: [0, 9]}')").is_none()); + assert!( + slice_zone("SELECT * FROM ARRAY_SLICE('img', '{row: [0, 9]}', '*', 0, NULL, NULL)") + .is_none() + ); + let zone = slice_zone(&format!( + "SELECT * FROM ARRAY_SLICE('img', '{{row: [0, 9]}}', '*', 0, {ZONE}, NULL)" + )) + .expect("zone"); + assert_eq!((zone.x_dim.as_str(), zone.y_dim.as_str()), ("col", "row")); + let zone = slice_zone( + "SELECT * FROM ARRAY_SLICE('img', '{row: [0, 9]}', NULL, NULL, \ + 'POLYGON((0 0, 4 0, 4 4, 0 4, 0 0))', ARRAY['col', 'row'])", + ) + .expect("zone"); + assert_eq!(zone.srid, None); + + let five = plan(&format!( + "SELECT * FROM ARRAY_SLICE('img', '{{row: [0, 9]}}', '*', 0, {ZONE})" + )) + .expect_err("five args refused"); + assert!(format!("{five}").contains("zone_dims"), "{five}"); + let seven = plan(&format!( + "SELECT * FROM ARRAY_SLICE('img', '{{row: [0, 9]}}', '*', 0, {ZONE}, NULL, 1)" + )) + .expect_err("seven args refused"); + assert!(format!("{seven}").contains("2..=6"), "{seven}"); + } + + #[test] + fn zone_dims_without_a_zone_is_refused() { + let err = plan( + "SELECT * FROM ARRAY_SLICE('img', '{row: [0, 9]}', '*', 0, NULL, ARRAY['col', 'row'])", + ) + .expect_err("refused"); + assert!(format!("{err}").contains("needs a zone_wkt"), "{err}"); + } } diff --git a/nodedb-sql/src/planner/array_georef.rs b/nodedb-sql/src/planner/array_georef.rs new file mode 100644 index 000000000..a7f3429a9 --- /dev/null +++ b/nodedb-sql/src/planner/array_georef.rs @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Plan-time checks for an array `GEOREF` clause. +//! +//! The same rules run again in the engine schema builder. This copy +//! reports each refusal with its own SQLSTATE class. +//! +//! - SRID outside the supported set: `22000`. +//! - X or Y names no dim: `42703`. +//! - X equals Y, wrong dim type, or bad coefficients: `22000`. + +use nodedb_types::geometry::Srid; + +use crate::error::{Result, SqlError}; +use crate::types_array::{ArrayDimAst, ArrayDimType, ArrayGeorefAst}; + +/// Check `georef` against the dims of array `array`. +pub fn check_georef(array: &str, georef: &ArrayGeorefAst, dims: &[ArrayDimAst]) -> Result<()> { + let data = |detail: String| SqlError::DataException { + detail: format!("array '{array}' GEOREF: {detail}"), + }; + Srid::try_from(georef.srid).map_err(|e| data(e.to_string()))?; + let dim = |name: &str| { + dims.iter() + .find(|d| d.name == name) + .ok_or_else(|| SqlError::UnknownColumn { + table: array.to_string(), + column: name.to_string(), + }) + }; + let x = dim(&georef.x_dim)?; + let y = dim(&georef.y_dim)?; + if x.name == y.name { + return Err(data(format!("X and Y name the same dim '{}'", x.name))); + } + match &georef.transform { + None => { + for (axis, d) in [("X", x), ("Y", y)] { + if !matches!(d.dtype, ArrayDimType::Int64 | ArrayDimType::Float64) { + return Err(data(format!( + "{axis} dim '{}' is {}; world coordinates need INT64 or FLOAT64", + d.name, + dim_type_name(d.dtype) + ))); + } + } + } + Some(gt) => { + for (axis, d) in [("X", x), ("Y", y)] { + if d.dtype != ArrayDimType::Int64 { + return Err(data(format!( + "{axis} dim '{}' is {}; pixel indices with TRANSFORM need INT64", + d.name, + dim_type_name(d.dtype) + ))); + } + } + if let Some(i) = gt.iter().position(|c| !c.is_finite()) { + return Err(data(format!( + "TRANSFORM coefficient {i} is {}; every coefficient must be finite", + gt[i] + ))); + } + let det = gt[1] * gt[5] - gt[2] * gt[4]; + if det == 0.0 || !det.is_finite() { + return Err(data(format!( + "TRANSFORM determinant GT1*GT5 - GT2*GT4 is {det}; it must be finite and non-zero" + ))); + } + } + } + Ok(()) +} + +fn dim_type_name(dtype: ArrayDimType) -> &'static str { + match dtype { + ArrayDimType::Int64 => "INT64", + ArrayDimType::Float64 => "FLOAT64", + ArrayDimType::TimestampMs => "TIMESTAMP_MS", + ArrayDimType::String => "STRING", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types_array::ArrayDomainBound; + + fn dim(name: &str, dtype: ArrayDimType) -> ArrayDimAst { + let (lo, hi) = match dtype { + ArrayDimType::Int64 => (ArrayDomainBound::Int64(0), ArrayDomainBound::Int64(9)), + ArrayDimType::Float64 => ( + ArrayDomainBound::Float64(0.0), + ArrayDomainBound::Float64(9.0), + ), + ArrayDimType::TimestampMs => ( + ArrayDomainBound::TimestampMs(0), + ArrayDomainBound::TimestampMs(9), + ), + ArrayDimType::String => ( + ArrayDomainBound::String(String::new()), + ArrayDomainBound::String("z".into()), + ), + }; + ArrayDimAst { + name: name.into(), + dtype, + lo, + hi, + } + } + + fn georef(transform: Option<[f64; 6]>) -> ArrayGeorefAst { + ArrayGeorefAst { + srid: 32647, + x_dim: "col".into(), + y_dim: "row".into(), + transform, + } + } + + const GT: [f64; 6] = [600_000.0, 10.0, 0.0, 700_000.0, 0.0, -10.0]; + + fn ints() -> Vec { + vec![ + dim("row", ArrayDimType::Int64), + dim("col", ArrayDimType::Int64), + ] + } + + #[test] + fn accepts_pixel_and_world() { + assert!(check_georef("a", &georef(Some(GT)), &ints()).is_ok()); + let world = vec![ + dim("row", ArrayDimType::Float64), + dim("col", ArrayDimType::Int64), + ]; + assert!(check_georef("a", &georef(None), &world).is_ok()); + } + + #[test] + fn unsupported_srid_is_22000() { + let mut g = georef(None); + g.srid = 2154; + let err = check_georef("a", &g, &ints()).unwrap_err(); + let SqlError::DataException { detail } = err else { + panic!("expected DataException, got {err:?}"); + }; + assert!( + detail.contains("4326, 32647, 32648, 3375, 3376"), + "{detail}" + ); + } + + #[test] + fn unknown_dim_is_42703() { + let mut g = georef(None); + g.y_dim = "lat".into(); + let err = check_georef("a", &g, &ints()).unwrap_err(); + assert_eq!( + err, + SqlError::UnknownColumn { + table: "a".into(), + column: "lat".into() + } + ); + } + + #[test] + fn same_dim_is_22000() { + let mut g = georef(None); + g.y_dim = "col".into(); + let err = check_georef("a", &g, &ints()).unwrap_err(); + assert!(matches!(err, SqlError::DataException { .. }), "{err:?}"); + } + + #[test] + fn wrong_dim_type_is_22000() { + let strings = vec![ + dim("row", ArrayDimType::String), + dim("col", ArrayDimType::Int64), + ]; + let err = check_georef("a", &georef(None), &strings).unwrap_err(); + assert!(matches!(err, SqlError::DataException { .. }), "{err:?}"); + let floats = vec![ + dim("row", ArrayDimType::Float64), + dim("col", ArrayDimType::Float64), + ]; + let err = check_georef("a", &georef(Some(GT)), &floats).unwrap_err(); + assert!(matches!(err, SqlError::DataException { .. }), "{err:?}"); + let times = vec![ + dim("row", ArrayDimType::TimestampMs), + dim("col", ArrayDimType::Int64), + ]; + let err = check_georef("a", &georef(None), ×).unwrap_err(); + assert!(matches!(err, SqlError::DataException { .. }), "{err:?}"); + } + + #[test] + fn bad_coefficients_are_22000() { + let mut inf = GT; + inf[0] = f64::INFINITY; + for gt in [ + inf, + [0.0, 1.0, 2.0, 0.0, 2.0, 4.0], + [0.0, 1e300, 0.0, 0.0, 0.0, 1e300], + ] { + let err = check_georef("a", &georef(Some(gt)), &ints()).unwrap_err(); + assert!(matches!(err, SqlError::DataException { .. }), "{err:?}"); + } + } +} diff --git a/nodedb-sql/src/planner/dml/bulk_load.rs b/nodedb-sql/src/planner/dml/bulk_load.rs index cc162cbdd..aadc3470d 100644 --- a/nodedb-sql/src/planner/dml/bulk_load.rs +++ b/nodedb-sql/src/planner/dml/bulk_load.rs @@ -49,6 +49,7 @@ mod tests { dims: Vec::new(), attrs: Vec::new(), tile_extents: Vec::new(), + georef: None, }) } } diff --git a/nodedb-sql/src/planner/mod.rs b/nodedb-sql/src/planner/mod.rs index df8c92932..34fa459e8 100644 --- a/nodedb-sql/src/planner/mod.rs +++ b/nodedb-sql/src/planner/mod.rs @@ -9,6 +9,7 @@ pub mod aggregate_order; pub mod array_ddl; pub mod array_dml; pub mod array_fn; +pub mod array_georef; pub mod ast_helpers; pub mod bitmap_emit; pub mod catalog_expr_fold; diff --git a/nodedb-sql/src/planner/select/array_cell_geom.rs b/nodedb-sql/src/planner/select/array_cell_geom.rs new file mode 100644 index 000000000..02acc723f --- /dev/null +++ b/nodedb-sql/src/planner/select/array_cell_geom.rs @@ -0,0 +1,439 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! `ARRAY_CELL_POINT()` and `ARRAY_CELL_FOOTPRINT()` in a SELECT over +//! `ARRAY_SLICE` / `ARRAY_PROJECT` of a georeferenced array. +//! +//! The planner rewrites each call into spatial functions over the cell's +//! GEOREF dim columns, with the GEOREF's SRID and coefficients as literals: +//! +//! - World mode, point: `ST_SetSRID(ST_Point(x, y), srid)`. +//! - Pixel mode, point: the affine of the pixel centre `(x + 0.5, y + 0.5)`. +//! - Pixel mode, footprint: `ST_SetSRID(ST_MakePolygon(..), srid)` over the +//! affine of the four pixel corners. +//! +//! Refusals: a footprint in world mode is `0A000`. Either call on an array +//! without GEOREF is `55000`. + +use std::ops::ControlFlow; + +use sqlparser::ast::{self, Select}; +use sqlparser::dialect::PostgreSqlDialect; +use sqlparser::parser::Parser; + +use crate::catalog::ArrayCatalogView; +use crate::error::{Result, SqlError}; +use crate::planner::array_fn::{ArrayTableFn, try_plan_array_table_fn}; +use crate::temporal::TemporalScope; +use crate::types::SqlCatalog; + +/// The SELECT and ORDER BY of a cell read with every cell geometry call +/// rewritten. +pub(super) struct CellGeomRewrite { + pub select: Select, + pub order_by: Option, +} + +/// Rewrite the cell geometry calls of a query whose FROM is one +/// `ARRAY_SLICE` / `ARRAY_PROJECT` call. +/// +/// The query's ORDER BY and SELECT list are read again after the SELECT body +/// plans, so the rewrite runs on the query before any of it plans. Returns +/// `None` when the query holds no call or reads no cells. +pub(in crate::planner::select) fn rewrite_cell_read_query( + select: &Select, + order_by: Option<&ast::OrderBy>, + catalog: &dyn SqlCatalog, + temporal: TemporalScope, +) -> Result> { + if !mentions_cell_geometry(select, order_by) { + return Ok(None); + } + let slice = match try_plan_array_table_fn(&select.from, catalog, temporal)? { + Some(ArrayTableFn::Slice(slice) | ArrayTableFn::Project(slice)) => slice, + Some(ArrayTableFn::Agg(_) | ArrayTableFn::Elementwise(_)) | None => return Ok(None), + }; + let view = catalog + .lookup_array(&slice.name) + .ok_or_else(|| SqlError::UnknownTable { + name: slice.name.clone(), + })?; + let qualifier = match &select.from[0].relation { + ast::TableFactor::Table { + alias: Some(alias), .. + } => alias.name.clone(), + _ => ast::Ident::with_quote('"', view.name.as_str()), + }; + rewrite_cell_geometry(select, order_by, &view, &qualifier) +} + +/// Whether `select` or `order_by` holds a cell geometry call. +fn mentions_cell_geometry(select: &Select, order_by: Option<&ast::OrderBy>) -> bool { + let has_call = |e: &ast::Expr| match cell_geom_kind(e) { + Some(_) => ControlFlow::Break(()), + None => ControlFlow::Continue(()), + }; + ast::visit_expressions(select, has_call).is_break() + || order_by.is_some_and(|o| ast::visit_expressions(o, has_call).is_break()) +} + +/// Rewrite the cell geometry calls in `select` and `order_by`. +/// +/// `qualifier` names the cell relation in the SELECT. Returns `None` when +/// neither clause holds a call. +pub(super) fn rewrite_cell_geometry( + select: &Select, + order_by: Option<&ast::OrderBy>, + view: &ArrayCatalogView, + qualifier: &ast::Ident, +) -> Result> { + if !mentions_cell_geometry(select, order_by) { + return Ok(None); + } + let mut rewrite = |expr: &mut ast::Expr| -> ControlFlow { + let Some(kind) = cell_geom_kind(expr) else { + return ControlFlow::Continue(()); + }; + match check_call(expr, kind).and_then(|()| replacement(kind, view, qualifier)) { + Ok(new) => { + *expr = new; + ControlFlow::Continue(()) + } + Err(e) => ControlFlow::Break(e), + } + }; + let mut select = select.clone(); + if let ControlFlow::Break(e) = ast::visit_expressions_mut(&mut select, &mut rewrite) { + return Err(e); + } + let mut order_by = order_by.cloned(); + if let Some(order) = order_by.as_mut() + && let ControlFlow::Break(e) = ast::visit_expressions_mut(order, &mut rewrite) + { + return Err(e); + } + Ok(Some(CellGeomRewrite { select, order_by })) +} + +/// Which cell geometry a call asks for. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum CellGeom { + Point, + Footprint, +} + +impl CellGeom { + fn sql_name(self) -> &'static str { + match self { + CellGeom::Point => "ARRAY_CELL_POINT", + CellGeom::Footprint => "ARRAY_CELL_FOOTPRINT", + } + } +} + +fn cell_geom_kind(expr: &ast::Expr) -> Option { + let ast::Expr::Function(f) = expr else { + return None; + }; + let [ast::ObjectNamePart::Identifier(ident)] = f.name.0.as_slice() else { + return None; + }; + if ident.value.eq_ignore_ascii_case("array_cell_point") { + Some(CellGeom::Point) + } else if ident.value.eq_ignore_ascii_case("array_cell_footprint") { + Some(CellGeom::Footprint) + } else { + None + } +} + +/// The call is a plain `NAME()`: no argument, no window, no filter. +fn check_call(expr: &ast::Expr, kind: CellGeom) -> Result<()> { + let ast::Expr::Function(f) = expr else { + return Ok(()); + }; + let no_args = match &f.args { + ast::FunctionArguments::List(list) => list.args.is_empty() && list.clauses.is_empty(), + ast::FunctionArguments::None | ast::FunctionArguments::Subquery(_) => false, + }; + if !no_args || f.over.is_some() || f.filter.is_some() || !f.within_group.is_empty() { + return Err(SqlError::Unsupported { + detail: format!( + "{}() takes no argument, window, or filter; it reads the cell of its row", + kind.sql_name() + ), + }); + } + Ok(()) +} + +/// The spatial expression a call rewrites to. +fn replacement( + kind: CellGeom, + view: &ArrayCatalogView, + qualifier: &ast::Ident, +) -> Result { + let georef = view + .georef + .as_ref() + .ok_or_else(|| SqlError::ObjectNotInPrerequisiteState { + object: format!("array '{}'", view.name), + detail: format!( + "{}(): array '{}' has no GEOREF; add one with ALTER ARRAY .. SET GEOREF", + kind.sql_name(), + view.name + ), + })?; + let x = column(qualifier, &georef.x_dim); + let y = column(qualifier, &georef.y_dim); + let srid = georef.srid; + let text = match (kind, georef.transform) { + (CellGeom::Point, None) => format!("ST_SetSRID(ST_Point({x}, {y}), {srid})"), + (CellGeom::Point, Some(gt)) => { + let [wx, wy] = affine(>, &format!("({x} + 5e-1)"), &format!("({y} + 5e-1)")); + format!("ST_SetSRID(ST_Point({wx}, {wy}), {srid})") + } + (CellGeom::Footprint, None) => { + return Err(SqlError::FeatureNotSupported { + detail: format!( + "ARRAY_CELL_FOOTPRINT() needs a GEOREF with TRANSFORM; array '{}' holds \ + world coordinates with no pixel size. Use ARRAY_CELL_POINT()", + view.name + ), + }); + } + (CellGeom::Footprint, Some(gt)) => { + let (u0, u1) = (x.clone(), format!("({x} + 1)")); + let (v0, v1) = (y.clone(), format!("({y} + 1)")); + let ring: Vec = [(&u0, &v0), (&u1, &v0), (&u1, &v1), (&u0, &v1), (&u0, &v0)] + .iter() + .map(|(u, v)| { + let [wx, wy] = affine(>, u, v); + format!("ARRAY[{wx}, {wy}]") + }) + .collect(); + format!( + "ST_SetSRID(ST_MakePolygon(ARRAY[{}]), {srid})", + ring.join(", ") + ) + } + }; + let dialect = PostgreSqlDialect {}; + Ok(Parser::new(&dialect).try_with_sql(&text)?.parse_expr()?) +} + +/// `qualifier."dim"`, with any `"` in the dim name doubled. +fn column(qualifier: &ast::Ident, dim: &str) -> String { + format!("{qualifier}.\"{}\"", dim.replace('"', "\"\"")) +} + +/// `[X, Y]` of the GDAL affine at the pixel position `(u, v)`, as SQL text. +/// The evaluation order matches `GeoTransform::pixel_to_world`. +fn affine(gt: &[f64; 6], u: &str, v: &str) -> [String; 2] { + let lit = |c: f64| format!("({c:e})"); + [ + format!( + "({} + {u} * {} + {v} * {})", + lit(gt[0]), + lit(gt[1]), + lit(gt[2]) + ), + format!( + "({} + {u} * {} + {v} * {})", + lit(gt[3]), + lit(gt[4]), + lit(gt[5]) + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::catalog::SqlCatalogError; + use crate::error::{Result, SqlError}; + use crate::functions::registry::FunctionRegistry; + use crate::parser::statement::parse_sql; + use crate::types::*; + use crate::types_array::{ + ArrayAttrAst, ArrayAttrType, ArrayDimAst, ArrayDimType, ArrayDomainBound, ArrayGeorefAst, + }; + + /// `img` is pixel mode, `world` is world mode, `plain` has no GEOREF. + struct GeoCatalog; + + impl SqlCatalog for GeoCatalog { + fn get_collection( + &self, + _: nodedb_types::DatabaseId, + _: &str, + ) -> std::result::Result, SqlCatalogError> { + Ok(None) + } + + fn lookup_array(&self, name: &str) -> Option { + let dim = |name: &str| ArrayDimAst { + name: name.into(), + dtype: ArrayDimType::Int64, + lo: ArrayDomainBound::Int64(0), + hi: ArrayDomainBound::Int64(15), + }; + let georef = match name { + "img" => Some(ArrayGeorefAst { + srid: 32647, + x_dim: "col".into(), + y_dim: "row".into(), + transform: Some([600_000.0, 30.0, 0.0, 700_000.0, 0.0, -30.0]), + }), + "world" => Some(ArrayGeorefAst { + srid: 32647, + x_dim: "col".into(), + y_dim: "row".into(), + transform: None, + }), + "plain" => None, + _ => return None, + }; + Some(ArrayCatalogView { + name: name.into(), + dims: vec![dim("row"), dim("col")], + attrs: vec![ArrayAttrAst { + name: "b1".into(), + dtype: ArrayAttrType::Float64, + nullable: true, + }], + tile_extents: vec![4, 4], + georef, + }) + } + } + + fn plan(sql: &str) -> Result { + let stmts = parse_sql(sql)?; + let sqlparser::ast::Statement::Query(q) = &stmts[0] else { + panic!("not a query: {sql}"); + }; + crate::planner::select::plan_query( + q, + &GeoCatalog, + &FunctionRegistry::new(), + crate::TemporalScope::default(), + ) + } + + fn rewritten_projection(sql: &str, array: &str) -> String { + let stmts = parse_sql(sql).unwrap(); + let sqlparser::ast::Statement::Query(q) = &stmts[0] else { + panic!("not a query"); + }; + let ast::SetExpr::Select(select) = q.body.as_ref() else { + panic!("not a select"); + }; + let view = GeoCatalog.lookup_array(array).unwrap(); + let out = rewrite_cell_geometry(select, None, &view, &ast::Ident::new("t")) + .unwrap() + .expect("rewritten"); + out.select.projection[0].to_string() + } + + #[test] + fn a_world_point_reads_the_dims() { + let text = rewritten_projection( + "SELECT ARRAY_CELL_POINT() FROM ARRAY_SLICE('world', '{row: [0, 3]}') AS t", + "world", + ); + assert_eq!(text, "ST_SetSRID(ST_Point(t.\"col\", t.\"row\"), 32647)"); + } + + #[test] + fn a_pixel_point_is_the_affine_of_the_centre() { + let text = rewritten_projection( + "SELECT array_cell_point() FROM ARRAY_SLICE('img', '{row: [0, 3]}') AS t", + "img", + ); + assert!(text.contains("(t.\"col\" + 5e-1) * (3e1)"), "{text}"); + assert!(text.contains("(t.\"row\" + 5e-1) * (-3e1)"), "{text}"); + assert!(text.contains("(6e5)") && text.contains("(7e5)"), "{text}"); + } + + #[test] + fn a_pixel_footprint_has_five_corners() { + let text = rewritten_projection( + "SELECT ARRAY_CELL_FOOTPRINT() FROM ARRAY_SLICE('img', '{row: [0, 3]}') AS t", + "img", + ); + assert!( + text.starts_with("ST_SetSRID(ST_MakePolygon(ARRAY["), + "{text}" + ); + assert_eq!(text.matches("ARRAY[(").count(), 5, "{text}"); + } + + #[test] + fn cell_geometry_plans_over_a_slice_and_a_project() { + for sql in [ + "SELECT row, col, ST_AsText(ST_Transform(ARRAY_CELL_POINT(), 4326)) AS p \ + FROM ARRAY_SLICE('img', '{row: [0, 3]}')", + "SELECT ARRAY_CELL_FOOTPRINT() AS f FROM ARRAY_PROJECT('img', ['b1'])", + "SELECT ARRAY_CELL_POINT() AS p FROM ARRAY_SLICE('world', '{row: [0, 3]}') \ + WHERE ST_X(ARRAY_CELL_POINT()) > 0", + ] { + plan(sql).unwrap_or_else(|e| panic!("{sql}: {e}")); + } + } + + #[test] + fn order_by_calls_are_rewritten_too() { + let stmts = parse_sql( + "SELECT b1 FROM ARRAY_SLICE('img', '{row: [0, 3]}') AS t \ + ORDER BY ST_X(ARRAY_CELL_POINT())", + ) + .unwrap(); + let sqlparser::ast::Statement::Query(q) = &stmts[0] else { + panic!("not a query"); + }; + let ast::SetExpr::Select(select) = q.body.as_ref() else { + panic!("not a select"); + }; + let view = GeoCatalog.lookup_array("img").unwrap(); + let out = rewrite_cell_geometry(select, q.order_by.as_ref(), &view, &ast::Ident::new("t")) + .unwrap() + .expect("rewritten"); + let order = out.order_by.expect("order by").to_string(); + assert!(order.contains("ST_SetSRID(ST_Point("), "{order}"); + assert!( + !order.to_ascii_lowercase().contains("array_cell_point"), + "{order}" + ); + } + + #[test] + fn a_world_footprint_is_feature_not_supported() { + let err = plan("SELECT ARRAY_CELL_FOOTPRINT() FROM ARRAY_SLICE('world', '{row: [0, 3]}')") + .expect_err("refused"); + assert!( + matches!(err, SqlError::FeatureNotSupported { .. }), + "{err:?}" + ); + } + + #[test] + fn cell_geometry_without_georef_is_object_not_in_prerequisite_state() { + for call in ["ARRAY_CELL_POINT()", "ARRAY_CELL_FOOTPRINT()"] { + let err = plan(&format!( + "SELECT {call} FROM ARRAY_SLICE('plain', '{{row: [0, 3]}}')" + )) + .expect_err("refused"); + assert!( + matches!(err, SqlError::ObjectNotInPrerequisiteState { .. }), + "{call}: {err:?}" + ); + } + } + + #[test] + fn an_argument_is_refused() { + let err = plan("SELECT ARRAY_CELL_POINT(row) FROM ARRAY_SLICE('img', '{row: [0, 3]}')") + .expect_err("refused"); + assert!(matches!(err, SqlError::Unsupported { .. }), "{err:?}"); + } +} diff --git a/nodedb-sql/src/planner/select/array_from.rs b/nodedb-sql/src/planner/select/array_from.rs index d4ddc0dda..d0b105a91 100644 --- a/nodedb-sql/src/planner/select/array_from.rs +++ b/nodedb-sql/src/planner/select/array_from.rs @@ -15,6 +15,7 @@ use sqlparser::ast::{self, Select}; +use super::array_cell_geom::rewrite_cell_geometry; use super::cte_catalog::CteCatalog; use super::helpers::{convert_projection, plain_table_factor}; use super::query_tail::QueryTail; @@ -68,7 +69,27 @@ fn plan_cell_read( let view = array_view(ctx.catalog, &slice.name)?; let read = RelationRead::new(ctx, fn_name, &view.name)?; let columns = array_slice_columns(&view, &slice); - let outer = read.plan_outer(ctx, columns)?; + // Cell geometry calls become spatial functions over the dim columns + // before the outer SELECT resolves its expressions. + let rewritten = rewrite_cell_geometry(ctx.select, ctx.tail.order_by, &view, &read.qualifier)?; + let outer = match &rewritten { + None => read.plan_outer(ctx, columns)?, + Some(rewrite) => { + let tail = QueryTail { + order_by: rewrite.order_by.as_ref(), + limit_clause: ctx.tail.limit_clause, + fetch: ctx.tail.fetch, + }; + let rewritten_ctx = OuterContext { + select: &rewrite.select, + catalog: ctx.catalog, + functions: ctx.functions, + tail: &tail, + statement_output: ctx.statement_output, + }; + read.plan_outer(&rewritten_ctx, columns)? + } + }; push_down_into_slice(&mut slice, &view, &outer.plan, &read.name); Ok(read.into_cte(SqlPlan::ArraySlice(slice), outer)) } @@ -403,6 +424,7 @@ mod tests { dims: vec![dim("x"), dim("y")], attrs: vec![attr("nir"), attr("red")], tile_extents: vec![2, 2], + georef: None, }) } } diff --git a/nodedb-sql/src/planner/select/entry/fixtures.rs b/nodedb-sql/src/planner/select/entry/fixtures.rs index d3d6cd336..49625db6a 100644 --- a/nodedb-sql/src/planner/select/entry/fixtures.rs +++ b/nodedb-sql/src/planner/select/entry/fixtures.rs @@ -155,6 +155,7 @@ impl SqlCatalog for TestCatalog { nullable: true, }], tile_extents: vec![1, 1_000_000], + georef: None, }) } else { None diff --git a/nodedb-sql/src/planner/select/entry/search.rs b/nodedb-sql/src/planner/select/entry/search.rs index d6e1b2f47..720736f3e 100644 --- a/nodedb-sql/src/planner/select/entry/search.rs +++ b/nodedb-sql/src/planner/select/entry/search.rs @@ -7,6 +7,7 @@ use super::pk_prefilter::apply_vector_pk_prefilter; use crate::error::Result; use crate::functions::registry::FunctionRegistry; use crate::planner::aggregate_cp_wrap::wrap_aggregate_cp_items; +use crate::planner::select::array_cell_geom::rewrite_cell_read_query; use crate::planner::select::limit::apply_limit; use crate::planner::select::order_by::{apply_order_by, try_hybrid_from_projection}; use crate::planner::select::query_tail::QueryTail; @@ -23,11 +24,18 @@ pub(super) fn plan_select_query( temporal: TemporalScope, statement_output: bool, ) -> Result { + // Cell geometry calls over an array cell read become spatial functions + // before the SELECT body, its ORDER BY and its SELECT-list wrap plan. + let rewritten = rewrite_cell_read_query(select, query.order_by.as_ref(), catalog, temporal)?; + let (select, query_order_by) = match &rewritten { + Some(rewrite) => (&rewrite.select, rewrite.order_by.as_ref()), + None => (select, query.order_by.as_ref()), + }; // ORDER BY / LIMIT belong to the query, not to its SELECT body, // but the scan planner needs them to pick an access path that can // honour them — so they travel down with the SELECT. let tail = QueryTail { - order_by: query.order_by.as_ref(), + order_by: query_order_by, limit_clause: &query.limit_clause, fetch: query.fetch.as_ref(), }; @@ -51,7 +59,7 @@ pub(super) fn plan_select_query( SqlPlan::Scan { collection, .. } => Some(collection.clone()), _ => None, }; - if let Some(order_by) = &query.order_by { + if let Some(order_by) = query_order_by { plan = apply_order_by(&plan, order_by, functions, &select.projection, &scope)?; } // Fall back to a SELECT-projection scan for hybrid-search and diff --git a/nodedb-sql/src/planner/select/mod.rs b/nodedb-sql/src/planner/select/mod.rs index ace371d69..e84fd5f62 100644 --- a/nodedb-sql/src/planner/select/mod.rs +++ b/nodedb-sql/src/planner/select/mod.rs @@ -6,6 +6,7 @@ //! search patterns (vector, text, hybrid, spatial) directly from the AST //! instead of reverse-engineering an optimizer's output. +mod array_cell_geom; mod array_from; mod comma_lateral; mod cte_catalog; diff --git a/nodedb-sql/src/planner/select/order_by/vector_join.rs b/nodedb-sql/src/planner/select/order_by/vector_join.rs index 6a2008415..5f734868c 100644 --- a/nodedb-sql/src/planner/select/order_by/vector_join.rs +++ b/nodedb-sql/src/planner/select/order_by/vector_join.rs @@ -7,6 +7,7 @@ //! from the array slice. This module isolates the AST inspection used by //! the trigger detector to recognise the fusion-eligible join shape. +use crate::temporal::TemporalScope; use crate::types::ArraySlicePlan; use crate::types::SqlPlan; @@ -27,27 +28,29 @@ pub(super) fn extract_vector_join_target( left: &SqlPlan, right: &SqlPlan, ) -> Option { - match (left, right) { - ( - SqlPlan::Scan { collection, .. }, - SqlPlan::ArraySlice(ArraySlicePlan { name, slice, .. }), - ) => Some(VectorJoinTarget { - vector_collection: collection.clone(), - array_prefilter: Some(crate::types::ArrayPrefilter { - array_name: name.clone(), - slice: slice.clone(), - }), - }), - ( - SqlPlan::ArraySlice(ArraySlicePlan { name, slice, .. }), - SqlPlan::Scan { collection, .. }, - ) => Some(VectorJoinTarget { - vector_collection: collection.clone(), - array_prefilter: Some(crate::types::ArrayPrefilter { - array_name: name.clone(), - slice: slice.clone(), - }), - }), - _ => None, + let (collection, slice) = match (left, right) { + (SqlPlan::Scan { collection, .. }, SqlPlan::ArraySlice(slice)) + | (SqlPlan::ArraySlice(slice), SqlPlan::Scan { collection, .. }) => (collection, slice), + _ => return None, + }; + // The prefilter carries the array name and box only. A slice that a + // zone, row cap or temporal scope also bounds plans as a plain join. + let ArraySlicePlan { + name, + slice, + limit, + zone, + temporal, + .. + } = slice; + if zone.is_some() || *limit != 0 || *temporal != TemporalScope::default() { + return None; } + Some(VectorJoinTarget { + vector_collection: collection.clone(), + array_prefilter: Some(crate::types::ArrayPrefilter { + array_name: name.clone(), + slice: slice.clone(), + }), + }) } diff --git a/nodedb-sql/src/types/plan/variants/array.rs b/nodedb-sql/src/types/plan/variants/array.rs index 4a3790bb4..ad68f5573 100644 --- a/nodedb-sql/src/types/plan/variants/array.rs +++ b/nodedb-sql/src/types/plan/variants/array.rs @@ -14,6 +14,8 @@ pub struct CreateArrayPlan { pub tile_extents: Vec, pub cell_order: types_array::ArrayCellOrderAst, pub tile_order: types_array::ArrayTileOrderAst, + /// `GEOREF (...)` clause, checked against `dims`. `None` = no georef. + pub georef: Option, /// Hilbert-prefix bits for vShard routing (1–16, default 8). pub prefix_bits: u8, /// Audit-retention horizon in milliseconds. `None` = non-bitemporal. @@ -31,6 +33,8 @@ pub struct AlterArrayPlan { pub audit_retain_ms: Option>, /// New value for `minimum_audit_retain_ms`. Cannot be NULL. pub minimum_audit_retain_ms: Option, + /// Georef change. `None` = georef unchanged. + pub georef: Option, } /// Payload of [`SqlPlan::InsertArray`](crate::types::SqlPlan::InsertArray). @@ -56,6 +60,8 @@ pub struct ArraySlicePlan { pub attr_projection: Vec, /// 0 = unlimited. pub limit: u32, + /// Polygon zone inside the box. `None` = the whole box. + pub zone: Option, /// Bitemporal qualifier. When both axes are `None` / `Any`, the Data /// Plane returns the live (current) state — the default fast path. /// Populated from `AS OF SYSTEM TIME` / `AS OF VALID TIME` clauses. diff --git a/nodedb-sql/src/types_array.rs b/nodedb-sql/src/types_array.rs index 2c3811bff..2ef706ab7 100644 --- a/nodedb-sql/src/types_array.rs +++ b/nodedb-sql/src/types_array.rs @@ -174,16 +174,42 @@ pub struct ArraySliceAst { pub dim_ranges: Vec, } -/// `ARRAY_AGG(...)` polygon zone over two numeric dims. +/// `ARRAY_AGG(...)` / `ARRAY_SLICE(...)` polygon zone over two numeric dims. /// /// `polygons` holds one entry per member polygon: its exterior ring, then /// its holes. Each ring is closed and lists `[x, y]` positions, `x` on /// `x_dim` and `y` on `y_dim`. +/// +/// `srid: None` means the positions are dim coordinates. `Some` means they +/// are in that SRID, and `x_dim` / `y_dim` are the array's GEOREF axes. #[derive(Debug, Clone, PartialEq)] pub struct ArrayZoneAst { pub polygons: Vec>>, pub x_dim: String, pub y_dim: String, + pub srid: Option, +} + +/// `GEOREF (SRID n, X dim, Y dim [, TRANSFORM (gt0, .., gt5)])` clause. +/// +/// `transform: None` means the dims hold pixel-centre world coordinates. +/// `Some` means the dims hold `INT64` pixel indices and the six +/// coefficients map them to world coordinates in GDAL order. +#[derive(Debug, Clone, PartialEq)] +pub struct ArrayGeorefAst { + pub srid: u32, + pub x_dim: String, + pub y_dim: String, + pub transform: Option<[f64; 6]>, +} + +/// Georef change carried by `ALTER ARRAY`. +#[derive(Debug, Clone, PartialEq)] +pub enum ArrayGeorefAlter { + /// `SET GEOREF (...)`: install or replace the georef. + Set(ArrayGeorefAst), + /// `DROP GEOREF`: remove the georef. + Drop, } /// Reducer carried on `SqlPlan::ArrayAgg`. Mirrors diff --git a/nodedb-sql/src/visitor/plan_visitor/args.rs b/nodedb-sql/src/visitor/plan_visitor/args.rs index e76c108d6..249176b1f 100644 --- a/nodedb-sql/src/visitor/plan_visitor/args.rs +++ b/nodedb-sql/src/visitor/plan_visitor/args.rs @@ -17,8 +17,8 @@ use crate::types::query::{ AggregateExpr, EngineType, JoinType, Projection, SortKey, SpatialPredicate, WindowSpec, }; use crate::types_array::{ - ArrayAttrAst, ArrayCellOrderAst, ArrayDimAst, ArrayReducerAst, ArraySliceAst, - ArrayTileOrderAst, ArrayZoneAst, + ArrayAttrAst, ArrayCellOrderAst, ArrayDimAst, ArrayGeorefAlter, ArrayGeorefAst, + ArrayReducerAst, ArraySliceAst, ArrayTileOrderAst, ArrayZoneAst, }; use crate::types_expr::{SqlExpr, SqlPayloadAtom, SqlValue}; use nodedb_types::vector_distance::DistanceMetric; @@ -259,11 +259,36 @@ pub struct CreateArrayVisitArgs<'a> { pub tile_extents: &'a [i64], pub cell_order: ArrayCellOrderAst, pub tile_order: ArrayTileOrderAst, + pub georef: Option<&'a ArrayGeorefAst>, pub prefix_bits: u8, pub audit_retain_ms: Option, pub minimum_audit_retain_ms: Option, } +/// Parameters for [`super::trait_def::PlanVisitor::alter_array`]. +pub struct AlterArrayVisitArgs<'a> { + pub name: &'a str, + /// `None` = unchanged, `Some(None)` = unregister, `Some(Some(v))` = set. + pub audit_retain_ms: Option>, + pub minimum_audit_retain_ms: Option, + /// Georef change. `None` = georef unchanged. + pub georef: Option<&'a ArrayGeorefAlter>, +} + +/// Parameters for [`super::trait_def::PlanVisitor::array_slice`]. +pub struct ArraySliceVisitArgs<'a> { + pub name: &'a str, + /// Coordinate box the slice reads. No dim range = the whole array. + pub slice: &'a ArraySliceAst, + /// Attribute names. Empty = all attrs. + pub attr_projection: &'a [String], + /// 0 = unlimited. + pub limit: u32, + /// Polygon zone inside the box. `None` = the whole box. + pub zone: Option<&'a ArrayZoneAst>, + pub temporal: &'a TemporalScope, +} + /// Parameters for [`super::trait_def::PlanVisitor::array_agg`]. pub struct ArrayAggVisitArgs<'a> { pub name: &'a str, diff --git a/nodedb-sql/src/visitor/plan_visitor/dispatch_rest.rs b/nodedb-sql/src/visitor/plan_visitor/dispatch_rest.rs index 5f7566bf7..feb5be807 100644 --- a/nodedb-sql/src/visitor/plan_visitor/dispatch_rest.rs +++ b/nodedb-sql/src/visitor/plan_visitor/dispatch_rest.rs @@ -8,9 +8,9 @@ //! satisfy the compiler that this match is exhaustive over the full //! `SqlPlan` enum without repeating their field lists. use super::args::{ - ArrayAggVisitArgs, CreateArrayVisitArgs, LateralLoopVisitArgs, LateralTopKVisitArgs, - MergeVisitArgs, VectorPrimaryDeleteVisitArgs, VectorPrimaryInsertVisitArgs, - VectorPrimaryUpdateVisitArgs, + AlterArrayVisitArgs, ArrayAggVisitArgs, ArraySliceVisitArgs, CreateArrayVisitArgs, + LateralLoopVisitArgs, LateralTopKVisitArgs, MergeVisitArgs, VectorPrimaryDeleteVisitArgs, + VectorPrimaryInsertVisitArgs, VectorPrimaryUpdateVisitArgs, }; use super::trait_def::PlanVisitor; use crate::types::SqlPlan; @@ -56,6 +56,7 @@ pub(super) fn dispatch_rest( tile_extents, cell_order, tile_order, + georef, prefix_bits, audit_retain_ms, minimum_audit_retain_ms, @@ -66,6 +67,7 @@ pub(super) fn dispatch_rest( tile_extents, cell_order: *cell_order, tile_order: *tile_order, + georef: georef.as_ref(), prefix_bits: *prefix_bits, audit_retain_ms: *audit_retain_ms, minimum_audit_retain_ms: *minimum_audit_retain_ms, @@ -75,7 +77,13 @@ pub(super) fn dispatch_rest( name, audit_retain_ms, minimum_audit_retain_ms, - }) => visitor.alter_array(name, *audit_retain_ms, *minimum_audit_retain_ms), + georef, + }) => visitor.alter_array(AlterArrayVisitArgs { + name, + audit_retain_ms: *audit_retain_ms, + minimum_audit_retain_ms: *minimum_audit_retain_ms, + georef: georef.as_ref(), + }), SqlPlan::InsertArray(InsertArrayPlan { name, rows }) => visitor.insert_array(name, rows), SqlPlan::DeleteArray(DeleteArrayPlan { name, coords }) => { visitor.delete_array(name, coords) @@ -85,8 +93,16 @@ pub(super) fn dispatch_rest( slice, attr_projection, limit, + zone, + temporal, + }) => visitor.array_slice(ArraySliceVisitArgs { + name, + slice, + attr_projection, + limit: *limit, + zone: zone.as_ref(), temporal, - }) => visitor.array_slice(name, slice, attr_projection, *limit, temporal), + }), SqlPlan::ArrayAgg(ArrayAggPlan { name, attr, diff --git a/nodedb-sql/src/visitor/plan_visitor/trait_def.rs b/nodedb-sql/src/visitor/plan_visitor/trait_def.rs index 166596363..5ad59f7e4 100644 --- a/nodedb-sql/src/visitor/plan_visitor/trait_def.rs +++ b/nodedb-sql/src/visitor/plan_visitor/trait_def.rs @@ -6,20 +6,20 @@ //! bundled params struct from [`super::args`] instead of raw positional arguments. use super::args::{ - AggregateVisitArgs, ArrayAggVisitArgs, CreateArrayVisitArgs, DocumentIndexLookupVisitArgs, - HybridSearchTripleVisitArgs, HybridSearchVisitArgs, InsertVisitArgs, JoinVisitArgs, - LateralLoopVisitArgs, LateralTopKVisitArgs, MergeVisitArgs, RecursiveScanVisitArgs, - RecursiveValueVisitArgs, ScanVisitArgs, SpatialScanVisitArgs, SubqueryVisitArgs, - TextSearchVisitArgs, TimeseriesScanVisitArgs, UpdateFromVisitArgs, UpsertVisitArgs, - VectorPrimaryDeleteVisitArgs, VectorPrimaryInsertVisitArgs, VectorPrimaryUpdateVisitArgs, - VectorSearchVisitArgs, + AggregateVisitArgs, AlterArrayVisitArgs, ArrayAggVisitArgs, ArraySliceVisitArgs, + CreateArrayVisitArgs, DocumentIndexLookupVisitArgs, HybridSearchTripleVisitArgs, + HybridSearchVisitArgs, InsertVisitArgs, JoinVisitArgs, LateralLoopVisitArgs, + LateralTopKVisitArgs, MergeVisitArgs, RecursiveScanVisitArgs, RecursiveValueVisitArgs, + ScanVisitArgs, SpatialScanVisitArgs, SubqueryVisitArgs, TextSearchVisitArgs, + TimeseriesScanVisitArgs, UpdateFromVisitArgs, UpsertVisitArgs, VectorPrimaryDeleteVisitArgs, + VectorPrimaryInsertVisitArgs, VectorPrimaryUpdateVisitArgs, VectorSearchVisitArgs, }; use crate::temporal::TemporalScope; use crate::types::SqlPlan; use crate::types::filter::Filter; use crate::types::plan::KvInsertIntent; use crate::types::query::EngineType; -use crate::types_array::{ArrayBinaryOpAst, ArrayCoordLiteral, ArrayInsertRow, ArraySliceAst}; +use crate::types_array::{ArrayBinaryOpAst, ArrayCoordLiteral, ArrayInsertRow}; use crate::types_expr::{SqlExpr, SqlValue}; /// Executor parity contract: every [`SqlPlan`] variant must be handled. @@ -239,12 +239,7 @@ pub trait PlanVisitor { fn drop_array(&mut self, name: &str, if_exists: bool) -> Result; /// Handle [`SqlPlan::AlterArray`]. - fn alter_array( - &mut self, - name: &str, - audit_retain_ms: Option>, - minimum_audit_retain_ms: Option, - ) -> Result; + fn alter_array(&mut self, args: AlterArrayVisitArgs<'_>) -> Result; /// Handle [`SqlPlan::InsertArray`]. fn insert_array( @@ -261,14 +256,7 @@ pub trait PlanVisitor { ) -> Result; /// Handle [`SqlPlan::ArraySlice`]. - fn array_slice( - &mut self, - name: &str, - slice: &ArraySliceAst, - attr_projection: &[String], - limit: u32, - temporal: &TemporalScope, - ) -> Result; + fn array_slice(&mut self, args: ArraySliceVisitArgs<'_>) -> Result; /// Handle [`SqlPlan::ArrayAgg`]. fn array_agg(&mut self, args: ArrayAggVisitArgs<'_>) -> Result; diff --git a/nodedb-test-support/src/array_sync.rs b/nodedb-test-support/src/array_sync.rs index 83da01d6d..e3e215afe 100644 --- a/nodedb-test-support/src/array_sync.rs +++ b/nodedb-test-support/src/array_sync.rs @@ -39,6 +39,7 @@ pub fn simple_schema(name: &str) -> ArraySchema { tile_extents: vec![10], cell_order: CellOrder::RowMajor, tile_order: TileOrder::RowMajor, + georef: None, } } diff --git a/nodedb-types/src/error/sqlstate.rs b/nodedb-types/src/error/sqlstate.rs index 050dace21..c2d1f6068 100644 --- a/nodedb-types/src/error/sqlstate.rs +++ b/nodedb-types/src/error/sqlstate.rs @@ -77,6 +77,10 @@ pub const INVALID_LIMIT_VALUE: &str = "2201W"; /// parse as the number an operation reads, e.g. `INCR` on `"abc"`) pub const INVALID_TEXT_REPRESENTATION: &str = "22P02"; +/// `22P04` — `bad_copy_file_format` (a `COPY` input file that does not +/// decode as its format, e.g. a truncated or corrupt GeoTIFF) +pub const BAD_COPY_FILE_FORMAT: &str = "22P04"; + /// `22023` — `invalid_parameter_value` (a `SET` value the parameter's own /// grammar refuses, e.g. `SET statement_timeout = 'later'`) pub const INVALID_PARAMETER_VALUE: &str = "22023"; @@ -418,6 +422,7 @@ mod tests { DIVISION_BY_ZERO, INVALID_LIMIT_VALUE, INVALID_TEXT_REPRESENTATION, + BAD_COPY_FILE_FORMAT, INTEGRITY_CONSTRAINT_VIOLATION, GENERATED_ALWAYS, NOT_NULL_VIOLATION, @@ -511,6 +516,7 @@ mod tests { assert_eq!(GROUPING_ERROR, "42803"); assert_eq!(UNDEFINED_TABLE, "42P01"); assert_eq!(INSUFFICIENT_PRIVILEGE, "42501"); + assert_eq!(BAD_COPY_FILE_FORMAT, "22P04"); assert_eq!(QUERY_CANCELED.0, "57014"); assert_eq!(INTERNAL_ERROR, "XX000"); assert_eq!(FEATURE_NOT_SUPPORTED, "0A000"); diff --git a/nodedb/Cargo.toml b/nodedb/Cargo.toml index ae5e6af77..633e841e6 100644 --- a/nodedb/Cargo.toml +++ b/nodedb/Cargo.toml @@ -40,6 +40,7 @@ nodedb-spatial = { workspace = true } nodedb-graph = { workspace = true } nodedb-vector = { workspace = true } nodedb-array = { workspace = true } +nodedb-raster = { workspace = true } nodedb-sql = { workspace = true } arrow = { workspace = true } @@ -179,6 +180,8 @@ nodedb-test-support = { workspace = true } rcgen = { workspace = true } zerompk = { workspace = true } loro = { workspace = true } +# Writes the GeoTIFF fixtures of the raster COPY wire tests. +tiff = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] # io_uring for Data Plane batched I/O (columnar reads, WAL) diff --git a/nodedb/src/control/array_catalog/ddl.rs b/nodedb/src/control/array_catalog/ddl.rs index 328b02c0d..64ed21fb8 100644 --- a/nodedb/src/control/array_catalog/ddl.rs +++ b/nodedb/src/control/array_catalog/ddl.rs @@ -17,6 +17,8 @@ use nodedb_types::Hlc; use crate::bridge::envelope::{Payload, PhysicalPlan, Response, Status}; use crate::control::array_catalog::ArrayCatalogEntry; +use crate::control::array_catalog::georef::check_same_content; +use crate::control::array_catalog::retention::check_retention_floor; use crate::control::catalog_entry::CatalogEntry; use crate::control::metadata_proposer::propose_catalog_batch_async; use crate::control::security::catalog::SystemCatalog; @@ -159,14 +161,35 @@ fn entry_for( array_id, audit_retain_ms, minimum_audit_retain_ms, + schema_msgpack, }) => { let current = committed(array_id)?.ok_or_else(|| crate::Error::PlanError { detail: format!("ALTER ARRAY {array_id}: not found"), })?; + check_retention_floor( + array_id, + ¤t, + *audit_retain_ms, + *minimum_audit_retain_ms, + )?; + // A georef change keeps every field `schema_hash` covers. Checked + // against the committed schema, so the kept hash stays true. + let schema_msgpack = match schema_msgpack { + Some(altered) => { + check_same_content( + &format!("ALTER ARRAY {array_id}"), + ¤t.schema_msgpack, + altered, + )?; + altered.clone() + } + None => current.schema_msgpack.clone(), + }; Ok(CatalogEntry::PutArray(Box::new(ArrayCatalogEntry { audit_retain_ms: audit_retain_ms.unwrap_or(current.audit_retain_ms), minimum_audit_retain_ms: minimum_audit_retain_ms .unwrap_or(current.minimum_audit_retain_ms), + schema_msgpack, ..current }))) } @@ -264,6 +287,7 @@ mod tests { array_id: "grid".to_string(), audit_retain_ms: Some(Some(60_000)), minimum_audit_retain_ms: None, + schema_msgpack: None, }); let CatalogEntry::PutArray(altered) = entry(catalog, &alter).expect("alter") else { unreachable!("ALTER ARRAY builds a put"); @@ -271,9 +295,104 @@ mod tests { assert_eq!(altered.audit_retain_ms, Some(60_000)); assert_eq!(altered.minimum_audit_retain_ms, None); assert_eq!(altered.schema_hash, created.schema_hash); + assert_eq!(altered.schema_msgpack, created.schema_msgpack); assert_eq!(altered.array_id, created.array_id); } + fn raster_schema() -> nodedb_array::schema::ArraySchema { + use nodedb_array::schema::{ArraySchemaBuilder, AttrSpec, AttrType, DimSpec, DimType}; + use nodedb_array::types::domain::{Domain, DomainBound}; + let dim = |name: &str| { + DimSpec::new( + name, + DimType::Int64, + Domain::new(DomainBound::Int64(0), DomainBound::Int64(1023)), + ) + }; + ArraySchemaBuilder::new("scene") + .dim(dim("row")) + .dim(dim("col")) + .attr(AttrSpec::new("b1", AttrType::UInt16, true)) + .tile_extents(vec![256, 256]) + .build() + .expect("raster schema") + } + + fn put_raster(catalog: &SystemCatalog) -> ArrayCatalogEntry { + let plan = PhysicalPlan::Array(ArrayOp::OpenArray { + array_id: ArrayId::in_database(TenantId::new(1), DatabaseId::DEFAULT, "scene"), + schema_msgpack: zerompk::to_msgpack_vec(&raster_schema()).expect("encode"), + schema_hash: 0xFEED, + prefix_bits: 8, + audit_retain_ms: Some(1_000), + minimum_audit_retain_ms: Some(10), + }); + let CatalogEntry::PutArray(created) = entry(catalog, &plan).expect("create") else { + unreachable!("CREATE ARRAY builds a put"); + }; + catalog.put_array(&created).expect("apply the create"); + *created + } + + fn alter_schema(schema_msgpack: Vec) -> PhysicalPlan { + PhysicalPlan::Meta(MetaOp::AlterArray { + array_id: "scene".to_string(), + audit_retain_ms: None, + minimum_audit_retain_ms: None, + schema_msgpack: Some(schema_msgpack), + }) + } + + /// A georef change replaces the schema bytes and keeps every other field. + #[test] + fn alter_georef_keeps_hash_and_every_other_field() { + let (store, _tmp) = catalog(); + let catalog = store.catalog(); + let created = put_raster(catalog); + let georef = nodedb_array::schema::ArrayGeoref { + srid: nodedb_types::geometry::Srid::Utm47N, + x_dim: 1, + y_dim: 0, + transform: Some(nodedb_array::schema::GeoTransform([ + 600_000.0, 10.0, 0.0, 700_000.0, 0.0, -10.0, + ])), + }; + let with_georef = raster_schema() + .with_georef(Some(georef.clone())) + .expect("georef"); + let bytes = zerompk::to_msgpack_vec(&with_georef).expect("encode"); + let CatalogEntry::PutArray(altered) = + entry(catalog, &alter_schema(bytes.clone())).expect("alter") + else { + unreachable!("ALTER ARRAY builds a put"); + }; + assert_eq!(altered.schema_msgpack, bytes); + assert_eq!(altered.schema_hash, created.schema_hash); + assert_eq!(altered.prefix_bits, created.prefix_bits); + assert_eq!(altered.audit_retain_ms, created.audit_retain_ms); + assert_eq!( + altered.minimum_audit_retain_ms, + created.minimum_audit_retain_ms + ); + assert_eq!(altered.array_id, created.array_id); + assert_eq!(altered.created_at_ms, created.created_at_ms); + let decoded: nodedb_array::schema::ArraySchema = + zerompk::from_msgpack(&altered.schema_msgpack).expect("decode"); + assert_eq!(decoded.georef, Some(georef)); + } + + /// The apply refuses a schema that changes a field `schema_hash` covers. + #[test] + fn alter_refuses_a_content_change() { + let (store, _tmp) = catalog(); + let catalog = store.catalog(); + put_raster(catalog); + let mut changed = raster_schema(); + changed.tile_extents = vec![128, 128]; + let bytes = zerompk::to_msgpack_vec(&changed).expect("encode"); + assert!(entry(catalog, &alter_schema(bytes)).is_err()); + } + #[test] fn only_array_ddl_takes_the_catalog_path() { assert!(is_array_ddl(&open_plan("grid"))); diff --git a/nodedb/src/control/array_catalog/entry.rs b/nodedb/src/control/array_catalog/entry.rs index b3fe8ce10..2be07a200 100644 --- a/nodedb/src/control/array_catalog/entry.rs +++ b/nodedb/src/control/array_catalog/entry.rs @@ -7,6 +7,8 @@ //! leak into Control-Plane catalog code. The Data Plane decodes the //! blob on open. +use std::hash::{DefaultHasher, Hash, Hasher}; + use nodedb_array::types::ArrayId; use serde::{Deserialize, Serialize}; @@ -60,6 +62,18 @@ pub struct ArrayCatalogEntry { pub incarnation: nodedb_types::Hlc, } +impl ArrayCatalogEntry { + /// The version a plan cache keys this array on. It changes with every + /// replicated put (CREATE, ALTER) and is never `0`, the "absent" value. + /// Both HLC parts count: a clock step back repeats `wall_ns` and moves + /// `logical`. Only equality is meaningful. + pub fn catalog_version(&self) -> u64 { + let mut hasher = DefaultHasher::new(); + (self.modification_hlc.wall_ns, self.modification_hlc.logical).hash(&mut hasher); + hasher.finish().max(1) + } +} + /// The target of a MOVE TENANT array rekey. #[derive(Debug, Clone, Copy, PartialEq, Eq, zerompk::ToMessagePack, zerompk::FromMessagePack)] pub struct ArrayMove { diff --git a/nodedb/src/control/array_catalog/georef.rs b/nodedb/src/control/array_catalog/georef.rs new file mode 100644 index 000000000..c817e4d3b --- /dev/null +++ b/nodedb/src/control/array_catalog/georef.rs @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Conversions between the engine georef and its SQL form. +//! +//! The engine names the X and Y axes by dim index. SQL names them by dim +//! name. Every refusal carries its own SQLSTATE class: `42703` for an +//! unknown dim, `22000` for every other georef error. + +use nodedb_array::ArrayError; +use nodedb_array::schema::{ArrayGeoref, ArraySchema, DimSpec, GeoTransform}; +use nodedb_sql::types_array::{ArrayGeorefAlter, ArrayGeorefAst}; +use nodedb_types::geometry::Srid; + +/// The SQL form of `schema`'s georef, axes named by dim. +/// +/// `None` when the schema has no georef, or when an axis index names no dim. +/// The schema builder refuses such an index, so a stored schema never holds one. +pub(crate) fn georef_ast(schema: &ArraySchema) -> Option { + let georef = schema.georef.as_ref()?; + let dim_name = |index: u32| { + usize::try_from(index) + .ok() + .and_then(|i| schema.dims.get(i)) + .map(|d| d.name.clone()) + }; + Some(ArrayGeorefAst { + srid: georef.srid.code(), + x_dim: dim_name(georef.x_dim)?, + y_dim: dim_name(georef.y_dim)?, + transform: georef.transform.map(|gt| gt.0), + }) +} + +/// The engine georef `ast` names, its axes resolved against `dims`. +pub(crate) fn georef_from_ast( + array: &str, + ast: &ArrayGeorefAst, + dims: &[DimSpec], +) -> crate::Result { + let srid = Srid::try_from(ast.srid).map_err(|e| crate::Error::DataException { + detail: format!("array '{array}' GEOREF: {e}"), + })?; + let index = |name: &str| { + dims.iter() + .position(|d| d.name == name) + .and_then(|i| u32::try_from(i).ok()) + .ok_or_else(|| crate::Error::UndefinedColumn { + column: name.to_string(), + }) + }; + Ok(ArrayGeoref { + srid, + x_dim: index(&ast.x_dim)?, + y_dim: index(&ast.y_dim)?, + transform: ast.transform.map(GeoTransform), + }) +} + +/// The stored schema `stored_msgpack` with `change` applied, re-encoded. +/// +/// The content bytes stay equal, so `schema_hash` and every segment stay valid. +pub(crate) fn regeoref_schema( + statement: &str, + stored_msgpack: &[u8], + change: &ArrayGeorefAlter, +) -> crate::Result> { + let schema = decode_schema(statement, stored_msgpack)?; + let georef = match change { + ArrayGeorefAlter::Set(ast) => Some(georef_from_ast(&schema.name, ast, &schema.dims)?), + ArrayGeorefAlter::Drop => None, + }; + let altered = schema + .with_georef(georef) + .map_err(|e| schema_error(statement, e))?; + let altered_msgpack = + zerompk::to_msgpack_vec(&altered).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("{statement}: schema encode: {e}"), + })?; + check_same_content(statement, stored_msgpack, &altered_msgpack)?; + Ok(altered_msgpack) +} + +/// Check that `altered_msgpack` keeps the content of `stored_msgpack`: every +/// field `schema_hash` covers. Only the georef may differ. +pub(crate) fn check_same_content( + statement: &str, + stored_msgpack: &[u8], + altered_msgpack: &[u8], +) -> crate::Result<()> { + let stored = decode_schema(statement, stored_msgpack)?; + let altered = decode_schema(statement, altered_msgpack)?; + if stored.name != altered.name || stored.content_msgpack() != altered.content_msgpack() { + return Err(crate::Error::Internal { + detail: format!( + "{statement}: the altered schema changes more than the georef; \ + schema_hash would no longer match" + ), + }); + } + Ok(()) +} + +fn decode_schema(statement: &str, msgpack: &[u8]) -> crate::Result { + zerompk::from_msgpack(msgpack).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("{statement}: array schema decode: {e}"), + }) +} + +/// The error a schema build or georef change reports. +/// +/// A georef refusal is `22000`. Every other schema refusal is a plan error. +pub(crate) fn schema_error(statement: &str, error: ArrayError) -> crate::Error { + match error { + ArrayError::InvalidGeoref { .. } => crate::Error::DataException { + detail: format!("{statement}: {error}"), + }, + other => crate::Error::PlanError { + detail: format!("{statement}: {other}"), + }, + } +} + +/// The clause body DESCRIBE shows, in the DDL form that recreates it. +pub(crate) fn georef_sql(ast: &ArrayGeorefAst) -> String { + let mut out = format!("SRID {}, X {}, Y {}", ast.srid, ast.x_dim, ast.y_dim); + if let Some(gt) = ast.transform { + let coefficients: Vec = gt.iter().map(|c| format!("{c:?}")).collect(); + out.push_str(&format!(", TRANSFORM ({})", coefficients.join(", "))); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_array::schema::{ArraySchemaBuilder, AttrSpec, AttrType, DimType}; + use nodedb_array::types::domain::{Domain, DomainBound}; + + fn dims() -> Vec { + ["row", "col"] + .into_iter() + .map(|name| { + DimSpec::new( + name, + DimType::Int64, + Domain::new(DomainBound::Int64(0), DomainBound::Int64(1023)), + ) + }) + .collect() + } + + fn ast() -> ArrayGeorefAst { + ArrayGeorefAst { + srid: 4326, + x_dim: "col".into(), + y_dim: "row".into(), + transform: Some([ + 100.5, + 8.983152841195214e-05, + 0.0, + 6.5, + 0.0, + -8.983152841195214e-05, + ]), + } + } + + #[test] + fn ast_round_trips_through_the_engine_georef() { + let georef = georef_from_ast("scene", &ast(), &dims()).unwrap(); + assert_eq!(georef.x_dim, 1); + assert_eq!(georef.y_dim, 0); + let mut builder = ArraySchemaBuilder::new("scene"); + for d in dims() { + builder = builder.dim(d); + } + let schema = builder + .attr(AttrSpec::new("b1", AttrType::UInt16, true)) + .tile_extents(vec![256, 256]) + .georef(georef) + .build() + .unwrap(); + assert_eq!(georef_ast(&schema), Some(ast())); + } + + #[test] + fn unknown_dim_is_undefined_column() { + let mut bad = ast(); + bad.x_dim = "lon".into(); + let err = georef_from_ast("scene", &bad, &dims()).unwrap_err(); + assert!( + matches!(err, crate::Error::UndefinedColumn { .. }), + "{err:?}" + ); + } + + #[test] + fn unsupported_srid_is_data_exception() { + let mut bad = ast(); + bad.srid = 2154; + let err = georef_from_ast("scene", &bad, &dims()).unwrap_err(); + assert!(matches!(err, crate::Error::DataException { .. }), "{err:?}"); + } + + #[test] + fn georef_refusal_is_data_exception() { + let err = schema_error( + "CREATE ARRAY scene", + ArrayError::InvalidGeoref { + array: "scene".into(), + detail: "singular".into(), + }, + ); + assert!(matches!(err, crate::Error::DataException { .. }), "{err:?}"); + } + + fn stored() -> ArraySchema { + let mut builder = ArraySchemaBuilder::new("scene"); + for d in dims() { + builder = builder.dim(d); + } + builder + .attr(AttrSpec::new("b1", AttrType::UInt16, true)) + .tile_extents(vec![256, 256]) + .build() + .unwrap() + } + + #[test] + fn regeoref_set_then_drop_keeps_content() { + let plain = zerompk::to_msgpack_vec(&stored()).unwrap(); + let set = + regeoref_schema("ALTER ARRAY scene", &plain, &ArrayGeorefAlter::Set(ast())).unwrap(); + let with: ArraySchema = zerompk::from_msgpack(&set).unwrap(); + assert_eq!(georef_ast(&with), Some(ast())); + assert_eq!(with.content_msgpack(), stored().content_msgpack()); + let dropped = regeoref_schema("ALTER ARRAY scene", &set, &ArrayGeorefAlter::Drop).unwrap(); + assert_eq!(dropped, plain); + } + + #[test] + fn regeoref_refusals_carry_their_class() { + let plain = zerompk::to_msgpack_vec(&stored()).unwrap(); + let mut unknown = ast(); + unknown.y_dim = "lat".into(); + let err = regeoref_schema("ALTER ARRAY scene", &plain, &ArrayGeorefAlter::Set(unknown)) + .unwrap_err(); + assert!( + matches!(err, crate::Error::UndefinedColumn { .. }), + "{err:?}" + ); + let mut singular = ast(); + singular.transform = Some([0.0, 1.0, 1.0, 0.0, 1.0, 1.0]); + let err = regeoref_schema( + "ALTER ARRAY scene", + &plain, + &ArrayGeorefAlter::Set(singular), + ) + .unwrap_err(); + assert!(matches!(err, crate::Error::DataException { .. }), "{err:?}"); + } + + #[test] + fn content_change_is_refused() { + let plain = zerompk::to_msgpack_vec(&stored()).unwrap(); + let mut changed = stored(); + changed.attrs[0] = AttrSpec::new("b1", AttrType::UInt8, true); + let changed = zerompk::to_msgpack_vec(&changed).unwrap(); + assert!(check_same_content("ALTER ARRAY scene", &plain, &changed).is_err()); + assert!(check_same_content("ALTER ARRAY scene", &plain, &plain).is_ok()); + } + + #[test] + fn sql_form_keeps_full_precision() { + let text = georef_sql(&ast()); + assert_eq!( + text, + "SRID 4326, X col, Y row, TRANSFORM (100.5, 8.983152841195213e-5, 0.0, 6.5, 0.0, \ + -8.983152841195213e-5)" + ); + // The printed form is the shortest that parses back to the same bits. + let printed = text + .split_once("TRANSFORM (") + .and_then(|(_, rest)| rest.strip_suffix(')')) + .expect("transform list"); + let parsed: Vec = printed + .split(", ") + .map(|c| c.parse().expect("coefficient")) + .collect(); + let original = ast().transform.expect("transform"); + assert_eq!(parsed.as_slice(), original.as_slice()); + } +} diff --git a/nodedb/src/control/array_catalog/mod.rs b/nodedb/src/control/array_catalog/mod.rs index 02124108b..2d1802b20 100644 --- a/nodedb/src/control/array_catalog/mod.rs +++ b/nodedb/src/control/array_catalog/mod.rs @@ -10,8 +10,10 @@ pub(crate) mod cell_route; pub(crate) mod ddl; pub mod entry; +pub(crate) mod georef; pub mod persist; pub mod registry; +pub(crate) mod retention; pub use entry::{ArrayCatalogEntry, ArrayMove}; pub use registry::{ArrayCatalog, ArrayCatalogHandle}; diff --git a/nodedb/src/control/array_catalog/retention.rs b/nodedb/src/control/array_catalog/retention.rs new file mode 100644 index 000000000..aac6f1a4c --- /dev/null +++ b/nodedb/src/control/array_catalog/retention.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The compliance floor of an ALTER ARRAY retention change. +//! +//! Lowering checks it for an early refusal. The catalog apply checks it again +//! against the committed entry, so a plan lowered before a concurrent or +//! later ALTER (a cached plan included) cannot shorten retention below the +//! floor. + +use nodedb_types::config::retention::BitemporalRetention; + +use super::ArrayCatalogEntry; + +/// Refuse an ALTER of `name` whose resulting `audit_retain_ms` falls below +/// its resulting `minimum_audit_retain_ms`. +/// +/// Both changes use ALTER's double-`Option`: `None` keeps the field of +/// `current`, `Some(None)` clears it, `Some(Some(v))` sets it. +pub(crate) fn check_retention_floor( + name: &str, + current: &ArrayCatalogEntry, + audit_retain_ms: Option>, + minimum_audit_retain_ms: Option>, +) -> crate::Result<()> { + let floor = match minimum_audit_retain_ms { + Some(changed) => changed.unwrap_or(0), + None => current.minimum_audit_retain_ms.unwrap_or(0), + }; + let retain = audit_retain_ms.unwrap_or(current.audit_retain_ms); + let Some(retain_ms) = retain else { + return Ok(()); + }; + let retention = BitemporalRetention { + data_retain_ms: 0, + audit_retain_ms: retain_ms as u64, + minimum_audit_retain_ms: floor, + }; + retention.validate().map_err(|e| crate::Error::PlanError { + detail: format!("ALTER ARRAY {name}: {e}"), + }) +} diff --git a/nodedb/src/control/cluster/array_cluster_exec/executor.rs b/nodedb/src/control/cluster/array_cluster_exec/executor.rs index b5f4a49dc..7caa85ca5 100644 --- a/nodedb/src/control/cluster/array_cluster_exec/executor.rs +++ b/nodedb/src/control/cluster/array_cluster_exec/executor.rs @@ -54,6 +54,7 @@ struct SliceArgs<'a> { prefix_bits: u8, system_time: nodedb_types::SystemTimeScope, valid_at_ms: Option, + zone_msgpack: &'a [u8], txn_id: Option, } @@ -121,6 +122,7 @@ impl ClusterArrayExecutor { prefix_bits, system_time, valid_at_ms, + zone_msgpack, } => { self.execute_slice(SliceArgs { array_id, @@ -131,6 +133,7 @@ impl ClusterArrayExecutor { prefix_bits: *prefix_bits, system_time: *system_time, valid_at_ms: *valid_at_ms, + zone_msgpack, txn_id, }) .await @@ -237,6 +240,7 @@ impl ClusterArrayExecutor { prefix_bits, system_time, valid_at_ms, + zone_msgpack, txn_id, } = args; let coordinator = ArrayCoordinator::for_slice( @@ -265,6 +269,7 @@ impl ClusterArrayExecutor { system_time, valid_at_ms, txn_id: txn_id.map(TxnId::as_u64), + zone_msgpack: zone_msgpack.to_vec(), }; let result = coordinator .coord_slice(req, limit, system_time) diff --git a/nodedb/src/control/cluster/array_executor/read.rs b/nodedb/src/control/cluster/array_executor/read.rs index abf0e1fec..91fc1c75e 100644 --- a/nodedb/src/control/cluster/array_executor/read.rs +++ b/nodedb/src/control/cluster/array_executor/read.rs @@ -51,6 +51,7 @@ impl DataPlaneArrayExecutor { shard_bucket: req.shard_bucket, system_time: req.system_time, valid_at_ms: req.valid_at_ms, + zone_msgpack: req.zone_msgpack.clone(), }); let resp = self diff --git a/nodedb/src/control/gateway/core.rs b/nodedb/src/control/gateway/core.rs index a198561e4..1c8ca3b78 100644 --- a/nodedb/src/control/gateway/core.rs +++ b/nodedb/src/control/gateway/core.rs @@ -430,64 +430,6 @@ impl Gateway { ) // routing_guard dropped here } - - /// Collect the descriptor version set for a plan using the current catalog. - /// - /// `tenant_id` must match the authenticated tenant of the query so that - /// the catalog key lookup (`"{tenant_id}:{collection_name}"`) finds the - /// correct descriptor version. Using tenant 0 here returns version 0 - /// for every collection stored under any other tenant, causing spurious - /// `DescriptorMismatch` rejections at the leader. - /// - /// `database_id` scopes the catalog lookup to the session's current database - /// so that a plan from one database cannot be served under another. - /// - /// Also folds in the tenant's permission-tree / RLS versions as pseudo - /// entries (see [`super::version_set::permission_tree_version_key`]). - pub(super) async fn collect_version_set( - &self, - plan: &PhysicalPlan, - tenant_id: u64, - database_id: DatabaseId, - ) -> Result { - let shared = self.shared()?; - let catalog = Some(shared.credentials.catalog()); - - let vs = GatewayVersionSet::from_plan(plan, |name| { - catalog - .and_then(|c| c.get_collection(database_id, tenant_id, name).ok()) - .flatten() - .map(|col| col.descriptor_version.max(1)) - .unwrap_or(0) - }); - - let permission_tree_version = shared - .permission_cache - .read() - .await - .tenant_version(tenant_id); - let rls_version = shared.rls.tenant_version(tenant_id); - - // A continuous aggregate read as a relation folds in its own catalog - // version, so a cached plan over it stops matching once it changes. - let vs = match catalog { - Some(catalog) => super::version_set::with_continuous_aggregates( - vs, - plan, - catalog, - database_id.as_u64(), - tenant_id, - ), - None => vs, - }; - - Ok(vs - .with_extra( - super::version_set::permission_tree_version_key(tenant_id), - permission_tree_version, - ) - .with_extra(super::version_set::rls_version_key(tenant_id), rls_version)) - } } #[cfg(test)] diff --git a/nodedb/src/control/gateway/mod.rs b/nodedb/src/control/gateway/mod.rs index d501bf33b..998f4acc4 100644 --- a/nodedb/src/control/gateway/mod.rs +++ b/nodedb/src/control/gateway/mod.rs @@ -24,6 +24,7 @@ pub mod router; pub mod sql_execute; pub mod stream; pub mod version_check; +pub mod version_collect; pub mod version_set; pub use core::Gateway; diff --git a/nodedb/src/control/gateway/sql_execute.rs b/nodedb/src/control/gateway/sql_execute.rs index 8da343f21..bccddd23d 100644 --- a/nodedb/src/control/gateway/sql_execute.rs +++ b/nodedb/src/control/gateway/sql_execute.rs @@ -14,7 +14,7 @@ use super::core::{Gateway, QueryContext, authorized_plan_for_context}; use super::lowered_plan::LoweredPlan; use super::plan_cache::{PlanCacheKey, SqlKey, hash_placeholder_types, hash_sql}; use super::version_set::{ - continuous_aggregate_of_key, permission_tree_version_key, rls_version_key, + array_of_key, continuous_aggregate_of_key, permission_tree_version_key, rls_version_key, }; impl Gateway { @@ -70,6 +70,14 @@ impl Gateway { if name == rls_key.as_str() { return rls_version; } + if let Some(array) = array_of_key(name, ctx.database_id.as_u64(), tenant_id) { + return super::version_set::array_version( + &shared.array_catalog, + ctx.database_id.as_u64(), + tenant_id, + array, + ); + } if let Some(aggregate) = continuous_aggregate_of_key(name, ctx.database_id.as_u64(), tenant_id) { diff --git a/nodedb/src/control/gateway/version_collect.rs b/nodedb/src/control/gateway/version_collect.rs new file mode 100644 index 000000000..f937e16e4 --- /dev/null +++ b/nodedb/src/control/gateway/version_collect.rs @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The version set a gateway plan is cached and checked under. + +use crate::Error; +use crate::types::DatabaseId; +use nodedb_physical::physical_plan::PhysicalPlan; + +use super::core::Gateway; +use super::version_set::{ + GatewayVersionSet, permission_tree_version_key, rls_version_key, with_arrays, + with_continuous_aggregates, +}; + +impl Gateway { + /// Collect the descriptor version set for a plan using the current catalog. + /// + /// `tenant_id` must match the authenticated tenant of the query so that + /// the catalog key lookup (`"{tenant_id}:{collection_name}"`) finds the + /// correct descriptor version. Using tenant 0 here returns version 0 + /// for every collection stored under any other tenant, causing spurious + /// `DescriptorMismatch` rejections at the leader. + /// + /// `database_id` scopes the catalog lookup to the session's current database + /// so that a plan from one database cannot be served under another. + /// + /// Also folds in the tenant's permission-tree / RLS versions as pseudo + /// entries (see [`permission_tree_version_key`]). + pub(super) async fn collect_version_set( + &self, + plan: &PhysicalPlan, + tenant_id: u64, + database_id: DatabaseId, + ) -> Result { + let shared = self.shared()?; + let catalog = Some(shared.credentials.catalog()); + + let vs = GatewayVersionSet::from_plan(plan, |name| { + catalog + .and_then(|c| c.get_collection(database_id, tenant_id, name).ok()) + .flatten() + .map(|col| col.descriptor_version.max(1)) + .unwrap_or(0) + }); + + let permission_tree_version = shared + .permission_cache + .read() + .await + .tenant_version(tenant_id); + let rls_version = shared.rls.tenant_version(tenant_id); + + // A continuous aggregate read as a relation folds in its own catalog + // version, so a cached plan over it stops matching once it changes. + let vs = match catalog { + Some(catalog) => { + with_continuous_aggregates(vs, plan, catalog, database_id.as_u64(), tenant_id) + } + None => vs, + }; + // An array read folds in its own catalog version, so a cached plan + // over it stops matching once the array is altered or dropped. + let vs = with_arrays( + vs, + plan, + &shared.array_catalog, + database_id.as_u64(), + tenant_id, + ); + + Ok(vs + .with_extra( + permission_tree_version_key(tenant_id), + permission_tree_version, + ) + .with_extra(rls_version_key(tenant_id), rls_version)) + } +} diff --git a/nodedb/src/control/gateway/version_set/array_keys.rs b/nodedb/src/control/gateway/version_set/array_keys.rs new file mode 100644 index 000000000..d4700ea01 --- /dev/null +++ b/nodedb/src/control/gateway/version_set/array_keys.rs @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The arrays a plan reads. +//! +//! Arrays live in their own catalog, so no array is a touched collection. +//! A cached plan over an array still carries its schema (georef, attrs, +//! retention), so it must stop matching once the array changes. + +use nodedb_array::types::ArrayId; +use nodedb_physical::physical_plan::{ArrayOp, ClusterArrayOp, PhysicalPlan}; +use nodedb_types::{DatabaseId, TenantId}; + +use super::GatewayVersionSet; +use super::keys::array_version_key; +use crate::control::array_catalog::ArrayCatalogHandle; +use crate::control::planner::plan_children::children; + +/// The catalog version of array `name` ([`ArrayCatalogEntry::catalog_version`]). +/// `0` when it does not exist. +/// +/// [`ArrayCatalogEntry::catalog_version`]: crate::control::array_catalog::ArrayCatalogEntry::catalog_version +pub fn array_version( + arrays: &ArrayCatalogHandle, + database_id: u64, + tenant_id: u64, + name: &str, +) -> u64 { + // A poisoned lock still holds a consistent mirror: every writer + // replaces whole entries. + let catalog = arrays + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + catalog + .lookup_by_name_in_database(TenantId::new(tenant_id), DatabaseId::new(database_id), name) + .map_or(0, |entry| entry.catalog_version()) +} + +/// `set` with one pseudo-entry per array `plan` reads. +pub fn with_arrays( + mut set: GatewayVersionSet, + plan: &PhysicalPlan, + arrays: &ArrayCatalogHandle, + database_id: u64, + tenant_id: u64, +) -> GatewayVersionSet { + for name in array_names(plan) { + let version = array_version(arrays, database_id, tenant_id, &name); + set = set.with_extra(array_version_key(database_id, tenant_id, &name), version); + } + set +} + +/// Every array name `plan` reads, at any depth. Both operands of an +/// elementwise op count. +pub fn array_names(plan: &PhysicalPlan) -> Vec { + let mut out = Vec::new(); + collect(plan, &mut out); + out +} + +fn collect(plan: &PhysicalPlan, out: &mut Vec) { + match plan { + PhysicalPlan::Array(op) => match op { + ArrayOp::Elementwise { left, right, .. } => push_pair(left, right, out), + other => out.push(other.primary_array().name.clone()), + }, + PhysicalPlan::ClusterArray(op) => match op { + ClusterArrayOp::Elementwise { left, right, .. } => push_pair(left, right, out), + other => out.push(other.array_id().name.clone()), + }, + _ => {} + } + for child in children(plan) { + collect(child, out); + } +} + +fn push_pair(left: &ArrayId, right: &ArrayId, out: &mut Vec) { + out.push(left.name.clone()); + out.push(right.name.clone()); +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_physical::physical_plan::{ExchangeMode, ExchangeOp, QueryOp}; + + fn aid(name: &str) -> ArrayId { + ArrayId::in_database(TenantId::new(1), DatabaseId::new(1), name) + } + + fn flush(name: &str) -> PhysicalPlan { + PhysicalPlan::Array(ArrayOp::Flush { + array_id: aid(name), + wal_lsn: 0, + }) + } + + #[test] + fn a_nested_array_op_is_found() { + let plan = PhysicalPlan::Query(QueryOp::Exchange(ExchangeOp { + child: Box::new(flush("scene")), + mode: ExchangeMode::Broadcast, + })); + assert_eq!(array_names(&plan), vec!["scene".to_string()]); + } + + #[test] + fn a_plan_without_arrays_names_none() { + let plan = PhysicalPlan::Query(QueryOp::ProviderScan { + provider: None, + rows: Vec::new(), + sources: Vec::new(), + filters: Vec::new(), + projection: Vec::new(), + computed_columns: Vec::new(), + window_functions: Vec::new(), + sort_keys: Vec::new(), + limit: None, + offset: 0, + distinct: false, + }); + assert!(array_names(&plan).is_empty()); + } +} diff --git a/nodedb/src/control/gateway/version_set/keys.rs b/nodedb/src/control/gateway/version_set/keys.rs index 6db697e73..e7d28c85a 100644 --- a/nodedb/src/control/gateway/version_set/keys.rs +++ b/nodedb/src/control/gateway/version_set/keys.rs @@ -42,6 +42,23 @@ pub fn continuous_aggregate_of_key(key: &str, database_id: u64, tenant_id: u64) .strip_prefix(&format!("{database_id}:{tenant_id}:")) } +/// Prefix for the synthetic entry carrying an array's catalog version. +const ARRAY_VERSION_KEY_PREFIX: &str = "\0__array::"; + +/// The synthetic pseudo-collection key an array's catalog version is folded +/// into the set under, so a plan over the array stops matching once the +/// array is altered, dropped, or re-created. +pub fn array_version_key(database_id: u64, tenant_id: u64, name: &str) -> String { + format!("{ARRAY_VERSION_KEY_PREFIX}{database_id}:{tenant_id}:{name}") +} + +/// The array name an [`array_version_key`] names, when `key` is one of +/// `database_id` / `tenant_id`. +pub fn array_of_key(key: &str, database_id: u64, tenant_id: u64) -> Option<&str> { + key.strip_prefix(ARRAY_VERSION_KEY_PREFIX)? + .strip_prefix(&format!("{database_id}:{tenant_id}:")) +} + #[cfg(test)] mod tests { use super::*; @@ -55,4 +72,16 @@ mod tests { assert_ne!(rls_version_key(1), rls_version_key(2)); assert_ne!(permission_tree_version_key(1), rls_version_key(1)); } + + #[test] + fn array_key_round_trips_only_in_its_scope() { + let key = array_version_key(7, 3, "scene"); + assert_eq!(array_of_key(&key, 7, 3), Some("scene")); + assert_eq!(array_of_key(&key, 7, 4), None); + assert_eq!(array_of_key(&key, 8, 3), None); + assert_eq!( + array_of_key(&continuous_aggregate_version_key(7, 3, "scene"), 7, 3), + None + ); + } } diff --git a/nodedb/src/control/gateway/version_set/mod.rs b/nodedb/src/control/gateway/version_set/mod.rs index fb418223d..1a19daca7 100644 --- a/nodedb/src/control/gateway/version_set/mod.rs +++ b/nodedb/src/control/gateway/version_set/mod.rs @@ -6,14 +6,16 @@ //! `PhysicalPlan` by walking every variant and extracting the collection //! name. +mod array_keys; mod keys; mod plan_keys; mod provider_keys; mod set; +pub use array_keys::{array_names, array_version, with_arrays}; pub use keys::{ - continuous_aggregate_of_key, continuous_aggregate_version_key, permission_tree_version_key, - rls_version_key, + array_of_key, array_version_key, continuous_aggregate_of_key, continuous_aggregate_version_key, + permission_tree_version_key, rls_version_key, }; pub use plan_keys::touched_collections; pub use provider_keys::{continuous_aggregate_version, provider_names, with_continuous_aggregates}; diff --git a/nodedb/src/control/planner/catalog_adapter/sql_catalog_impl.rs b/nodedb/src/control/planner/catalog_adapter/sql_catalog_impl.rs index 2ea36bea9..8fcf6a2cf 100644 --- a/nodedb/src/control/planner/catalog_adapter/sql_catalog_impl.rs +++ b/nodedb/src/control/planner/catalog_adapter/sql_catalog_impl.rs @@ -313,6 +313,21 @@ impl SqlCatalog for OriginCatalog { name, )? }; + // A plan built from this view (its georef, attrs, schema hash) is + // cached against the array's version, so an ALTER, DROP, or + // re-CREATE stops it matching. + self.recorded_versions + .lock() + .unwrap_or_else(|p| p.into_inner()) + .record_unleased( + DescriptorId::new( + self.database_id.as_u64(), + self.tenant_id, + DescriptorKind::Array, + entry.name.clone(), + ), + entry.catalog_version(), + ); let schema: ArraySchema = zerompk::from_msgpack(&entry.schema_msgpack).ok()?; let dims = schema @@ -355,11 +370,13 @@ impl SqlCatalog for OriginCatalog { } } + let georef = crate::control::array_catalog::georef::georef_ast(&schema); Some(ArrayCatalogView { name: schema.name, dims, attrs, tile_extents, + georef, }) } } diff --git a/nodedb/src/control/planner/context/query/planning_built.rs b/nodedb/src/control/planner/context/query/planning_built.rs index 75a1b961c..a12aff226 100644 --- a/nodedb/src/control/planner/context/query/planning_built.rs +++ b/nodedb/src/control/planner/context/query/planning_built.rs @@ -22,7 +22,7 @@ pub enum BulkLoadRoute { /// Each row loads as one `INSERT` statement. RowInserts, /// The rows decode against this array schema and load in cell chunks. - ArrayCells(ArrayCatalogView), + ArrayCells(Box), } impl QueryContext { @@ -47,7 +47,7 @@ impl QueryContext { BulkLoadPlan::RowInserts => Ok(BulkLoadRoute::RowInserts), BulkLoadPlan::CellChunks(_) => catalog .lookup_array(collection) - .map(BulkLoadRoute::ArrayCells) + .map(|view| BulkLoadRoute::ArrayCells(Box::new(view))) .ok_or_else(|| nodedb_sql::SqlError::Unsupported { detail: format!("COPY: array '{collection}' was dropped"), }), diff --git a/nodedb/src/control/planner/descriptor_set.rs b/nodedb/src/control/planner/descriptor_set.rs index a3525b2cc..c23849543 100644 --- a/nodedb/src/control/planner/descriptor_set.rs +++ b/nodedb/src/control/planner/descriptor_set.rs @@ -35,6 +35,11 @@ use nodedb_cluster::DescriptorId; pub struct DescriptorVersionSet { /// Sorted by `(kind, database_id, tenant_id, name)`. entries: Vec<(DescriptorId, u64)>, + /// Descriptors the plan read without a lease, at their catalog content + /// version. Same order. An array is one: its lease version is fixed, + /// so only this entry changes when an ALTER changes the schema a plan + /// embeds. Lease admission never sees these. + unleased: Vec<(DescriptorId, u64)>, /// The tenant's permission-tree version observed at plan-build time. /// `0` when no permission cache was consulted. Compared in /// [`Self::all_fresh`] alongside the descriptor entries so a revoked @@ -91,18 +96,14 @@ impl DescriptorVersionSet { /// snapshots, which is a planning bug. In release builds /// the later value wins. pub fn record(&mut self, id: DescriptorId, version: u64) { - match self.entries.binary_search_by(|(k, _)| cmp_id(k, &id)) { - Ok(idx) => { - debug_assert_eq!( - self.entries[idx].1, version, - "planner observed two versions of the same descriptor in one plan" - ); - self.entries[idx].1 = version; - } - Err(idx) => { - self.entries.insert(idx, (id, version)); - } - } + insert_sorted(&mut self.entries, id, version); + } + + /// Record a descriptor read without a lease at its content `version`. + /// Same ordering and duplicate rules as [`Self::record`]. Only the + /// freshness check reads it. + pub fn record_unleased(&mut self, id: DescriptorId, version: u64) { + insert_sorted(&mut self.unleased, id, version); } /// Iterate `(id, version)` pairs in stable order. @@ -136,7 +137,32 @@ impl DescriptorVersionSet { self.permission_tree_version == policies.permission_tree && self.rls_version == policies.rls && self.redaction_version == policies.redaction - && self.entries.iter().all(|(id, v)| current(id) == Some(*v)) + && self + .entries + .iter() + .chain(&self.unleased) + .all(|(id, v)| current(id) == Some(*v)) + } +} + +/// Insert `(id, version)` into the sorted `entries`. A repeat of the same +/// `id` keeps one entry. +/// +/// Panics in debug builds if `id` repeats with a different version: the +/// planner read the same descriptor twice under different catalog snapshots, +/// which is a planning bug. In release builds the later value wins. +fn insert_sorted(entries: &mut Vec<(DescriptorId, u64)>, id: DescriptorId, version: u64) { + match entries.binary_search_by(|(k, _)| cmp_id(k, &id)) { + Ok(idx) => { + debug_assert_eq!( + entries[idx].1, version, + "planner observed two versions of the same descriptor in one plan" + ); + entries[idx].1 = version; + } + Err(idx) => { + entries.insert(idx, (id, version)); + } } } @@ -246,6 +272,17 @@ mod tests { assert!(!set.all_fresh(lookup, policies(0, 0, 0))); } + #[test] + fn unleased_entries_gate_freshness_but_not_iteration() { + let array = DescriptorId::new(0, 1, DescriptorKind::Array, "scene".to_string()); + let mut set = DescriptorVersionSet::new(); + set.record_unleased(array.clone(), 7); + assert!(set.is_empty(), "lease admission iterates no unleased entry"); + assert!(set.all_fresh(|_| Some(7), policies(0, 0, 0))); + assert!(!set.all_fresh(|_| Some(8), policies(0, 0, 0))); + assert!(!set.all_fresh(|_| None, policies(0, 0, 0))); + } + #[test] fn all_fresh_false_on_missing_descriptor() { let mut set = DescriptorVersionSet::new(); diff --git a/nodedb/src/control/planner/mod.rs b/nodedb/src/control/planner/mod.rs index fb1e19a81..a37045775 100644 --- a/nodedb/src/control/planner/mod.rs +++ b/nodedb/src/control/planner/mod.rs @@ -8,6 +8,7 @@ pub mod descriptor_set; pub mod implicit_edges; pub mod materialized_sum; pub mod period_lock; +pub mod plan_children; pub(crate) mod plan_error_map; pub mod procedural; pub mod redaction_refusal; diff --git a/nodedb/src/control/planner/plan_children/document.rs b/nodedb/src/control/planner/plan_children/document.rs new file mode 100644 index 000000000..d97f92dbf --- /dev/null +++ b/nodedb/src/control/planner/plan_children/document.rs @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The child plan a document op reads its rows from. + +use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan}; + +/// The child plan `op` reads its rows from, if any. +pub(super) fn child(op: &DocumentOp) -> Option<&PhysicalPlan> { + match op { + DocumentOp::InsertSelect { source, .. } + | DocumentOp::Merge { source, .. } + | DocumentOp::UpdateFromJoin { source, .. } => source.query_plan(), + DocumentOp::ResolveWrite(inner) => child(inner), + DocumentOp::PointGet { .. } + | DocumentOp::PointPut { .. } + | DocumentOp::PointInsert { .. } + | DocumentOp::PointDelete { .. } + | DocumentOp::PointUpdate { .. } + | DocumentOp::Scan { .. } + | DocumentOp::BatchInsert { .. } + | DocumentOp::RangeScan { .. } + | DocumentOp::Register { .. } + | DocumentOp::IndexLookup { .. } + | DocumentOp::IndexedFetch { .. } + | DocumentOp::DropIndex { .. } + | DocumentOp::BackfillIndex { .. } + | DocumentOp::Truncate { .. } + | DocumentOp::EstimateCount { .. } + | DocumentOp::Upsert { .. } + | DocumentOp::BatchUpsert { .. } + | DocumentOp::BulkUpdate { .. } + | DocumentOp::BulkDelete { .. } + | DocumentOp::MaterializeScan { .. } + | DocumentOp::ApplyBalanceDelta { .. } + | DocumentOp::ResolvedWrite { .. } => None, + } +} diff --git a/nodedb/src/control/planner/plan_children/mod.rs b/nodedb/src/control/planner/plan_children/mod.rs new file mode 100644 index 000000000..962edf6d9 --- /dev/null +++ b/nodedb/src/control/planner/plan_children/mod.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The child plans of a `PhysicalPlan`, shared by every whole-plan walk. + +mod document; +mod query; +mod vector; +mod walk; + +pub use walk::children; diff --git a/nodedb/src/control/planner/plan_children/query.rs b/nodedb/src/control/planner/plan_children/query.rs new file mode 100644 index 000000000..bbce07040 --- /dev/null +++ b/nodedb/src/control/planner/plan_children/query.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The child plans a query op reads its rows from. + +use nodedb_physical::physical_plan::{PhysicalPlan, QueryOp}; + +/// The child plans `op` reads its rows from. +pub(super) fn children(op: &QueryOp) -> Vec<&PhysicalPlan> { + match op { + QueryOp::Exchange(exchange) => vec![&*exchange.child], + QueryOp::PostProcess { input, .. } => vec![&**input], + QueryOp::SetOp { inputs, .. } => inputs.iter().collect(), + QueryOp::Aggregate { input, .. } | QueryOp::PartialAggregateState { input, .. } => { + input.iter().map(|child| &**child).collect() + } + QueryOp::HashJoin { + left_input, + right_input, + left_bitmap, + right_bitmap, + .. + } => [left_input, right_input, left_bitmap, right_bitmap] + .into_iter() + .flatten() + .map(|child| &**child) + .collect(), + QueryOp::NestedLoopJoin { + left_input, + right_input, + .. + } + | QueryOp::SortMergeJoin { + left_input, + right_input, + .. + } => [left_input, right_input] + .into_iter() + .flatten() + .map(|child| &**child) + .collect(), + QueryOp::LateralTopK { outer_plan, .. } | QueryOp::LateralLoop { outer_plan, .. } => { + vec![&**outer_plan] + } + QueryOp::ProviderScan { .. } + | QueryOp::PartialAggregate { .. } + | QueryOp::ShuffleJoinConsume { .. } + | QueryOp::ShuffleAggregateConsume { .. } + | QueryOp::FacetCounts { .. } + | QueryOp::RecursiveScan { .. } + | QueryOp::RecursiveValue { .. } => Vec::new(), + } +} diff --git a/nodedb/src/control/planner/plan_children/vector.rs b/nodedb/src/control/planner/plan_children/vector.rs new file mode 100644 index 000000000..f2f8a6bf3 --- /dev/null +++ b/nodedb/src/control/planner/plan_children/vector.rs @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The sub-plan a vector op takes its prefilter from. + +use nodedb_physical::physical_plan::{PhysicalPlan, VectorOp}; + +/// The resolved sub-plan a vector search takes its prefilter from, if any. +pub(super) fn vector_child(op: &VectorOp) -> Option<&PhysicalPlan> { + match op { + VectorOp::Search { + inline_prefilter_plan: Some(child), + .. + } => Some(child), + VectorOp::ResolveDirectWrite(inner) => vector_child(inner), + VectorOp::Search { .. } + | VectorOp::Insert { .. } + | VectorOp::BatchInsert { .. } + | VectorOp::MultiSearch { .. } + | VectorOp::Delete { .. } + | VectorOp::DeleteBySurrogate { .. } + | VectorOp::SetParams { .. } + | VectorOp::DropIndex { .. } + | VectorOp::QueryStats { .. } + | VectorOp::Seal { .. } + | VectorOp::CompactIndex { .. } + | VectorOp::Rebuild { .. } + | VectorOp::SparseInsert { .. } + | VectorOp::SparseSearch { .. } + | VectorOp::SparseDelete { .. } + | VectorOp::MultiVectorInsert { .. } + | VectorOp::MultiVectorDelete { .. } + | VectorOp::MultiVectorScoreSearch { .. } + | VectorOp::DirectUpsert { .. } + | VectorOp::DirectInsert { .. } + | VectorOp::DirectInsertIfAbsent { .. } + | VectorOp::DirectBatchWrite { .. } + | VectorOp::DirectDelete { .. } + | VectorOp::DirectTruncate { .. } + | VectorOp::DirectUpdate { .. } + | VectorOp::ResolvedDirectWrite { .. } => None, + } +} diff --git a/nodedb/src/control/planner/plan_children/walk.rs b/nodedb/src/control/planner/plan_children/walk.rs new file mode 100644 index 000000000..a5e74ccd8 --- /dev/null +++ b/nodedb/src/control/planner/plan_children/walk.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The plans a plan reads its rows from, one level down. + +use nodedb_physical::physical_plan::PhysicalPlan; + +use super::{document, query, vector}; + +/// The plans whose rows `plan` reads. Exhaustive over [`PhysicalPlan`]. +pub fn children(plan: &PhysicalPlan) -> Vec<&PhysicalPlan> { + match plan { + PhysicalPlan::Query(op) => query::children(op), + PhysicalPlan::Document(op) => document::child(op).into_iter().collect(), + PhysicalPlan::Vector(op) => vector::vector_child(op).into_iter().collect(), + PhysicalPlan::Graph(_) + | PhysicalPlan::Kv(_) + | PhysicalPlan::Text(_) + | PhysicalPlan::Columnar(_) + | PhysicalPlan::Timeseries(_) + | PhysicalPlan::Spatial(_) + | PhysicalPlan::Crdt(_) + | PhysicalPlan::Meta(_) + | PhysicalPlan::Array(_) + | PhysicalPlan::ClusterArray(_) + | PhysicalPlan::ClusterEvent(_) => Vec::new(), + } +} diff --git a/nodedb/src/control/planner/plan_error_map.rs b/nodedb/src/control/planner/plan_error_map.rs index c4b01a807..f9910913c 100644 --- a/nodedb/src/control/planner/plan_error_map.rs +++ b/nodedb/src/control/planner/plan_error_map.rs @@ -90,11 +90,10 @@ pub(crate) fn map_plan_error( // Refusals of a constraint or clause NodeDB does not implement. The // DDL router renders both as `0A000`, so the planner path does too. nodedb_sql::SqlError::UnsupportedConstraint { .. } - | nodedb_sql::SqlError::ConflictingEngineClause { .. } => { - crate::Error::FeatureNotSupported { - detail: error.to_string(), - } - } + | nodedb_sql::SqlError::ConflictingEngineClause { .. } + | nodedb_sql::SqlError::FeatureNotSupported { .. } => crate::Error::FeatureNotSupported { + detail: error.to_string(), + }, // A value out of range for its type is `22003` // (numeric_value_out_of_range), the SQLSTATE PostgreSQL and the DDL // DEFAULT gate give it. diff --git a/nodedb/src/control/planner/redaction_refusal/expr_reads/document.rs b/nodedb/src/control/planner/redaction_refusal/expr_reads/document.rs index a91792513..38f5d5353 100644 --- a/nodedb/src/control/planner/redaction_refusal/expr_reads/document.rs +++ b/nodedb/src/control/planner/redaction_refusal/expr_reads/document.rs @@ -2,8 +2,8 @@ //! Column reads of the document engine's operations. +use nodedb_physical::physical_plan::DocumentOp; use nodedb_physical::physical_plan::document::MergeActionOp; -use nodedb_physical::physical_plan::{DocumentOp, PhysicalPlan}; use super::reads::{Clause, NodeReads}; @@ -104,35 +104,3 @@ pub(super) fn reads(op: &DocumentOp, out: &mut NodeReads) -> crate::Result<()> { } Ok(()) } - -/// The child plan `op` reads its rows from, if any. -pub(super) fn child(op: &DocumentOp) -> Option<&PhysicalPlan> { - match op { - DocumentOp::InsertSelect { source, .. } - | DocumentOp::Merge { source, .. } - | DocumentOp::UpdateFromJoin { source, .. } => source.query_plan(), - DocumentOp::ResolveWrite(inner) => child(inner), - DocumentOp::PointGet { .. } - | DocumentOp::PointPut { .. } - | DocumentOp::PointInsert { .. } - | DocumentOp::PointDelete { .. } - | DocumentOp::PointUpdate { .. } - | DocumentOp::Scan { .. } - | DocumentOp::BatchInsert { .. } - | DocumentOp::RangeScan { .. } - | DocumentOp::Register { .. } - | DocumentOp::IndexLookup { .. } - | DocumentOp::IndexedFetch { .. } - | DocumentOp::DropIndex { .. } - | DocumentOp::BackfillIndex { .. } - | DocumentOp::Truncate { .. } - | DocumentOp::EstimateCount { .. } - | DocumentOp::Upsert { .. } - | DocumentOp::BatchUpsert { .. } - | DocumentOp::BulkUpdate { .. } - | DocumentOp::BulkDelete { .. } - | DocumentOp::MaterializeScan { .. } - | DocumentOp::ApplyBalanceDelta { .. } - | DocumentOp::ResolvedWrite { .. } => None, - } -} diff --git a/nodedb/src/control/planner/redaction_refusal/expr_reads/query.rs b/nodedb/src/control/planner/redaction_refusal/expr_reads/query.rs index 9a4688bc4..3cf177553 100644 --- a/nodedb/src/control/planner/redaction_refusal/expr_reads/query.rs +++ b/nodedb/src/control/planner/redaction_refusal/expr_reads/query.rs @@ -2,7 +2,7 @@ //! Column reads and child plans of the relational operations. -use nodedb_physical::physical_plan::{JoinProjection, PhysicalPlan, QueryOp}; +use nodedb_physical::physical_plan::{JoinProjection, QueryOp}; use super::reads::NodeReads; @@ -143,50 +143,3 @@ fn join_renames(projection: &[JoinProjection], out: &mut NodeReads) { } } } - -/// The child plans `op` reads its rows from. -pub(super) fn children(op: &QueryOp) -> Vec<&PhysicalPlan> { - match op { - QueryOp::Exchange(exchange) => vec![&*exchange.child], - QueryOp::PostProcess { input, .. } => vec![&**input], - QueryOp::SetOp { inputs, .. } => inputs.iter().collect(), - QueryOp::Aggregate { input, .. } | QueryOp::PartialAggregateState { input, .. } => { - input.iter().map(|child| &**child).collect() - } - QueryOp::HashJoin { - left_input, - right_input, - left_bitmap, - right_bitmap, - .. - } => [left_input, right_input, left_bitmap, right_bitmap] - .into_iter() - .flatten() - .map(|child| &**child) - .collect(), - QueryOp::NestedLoopJoin { - left_input, - right_input, - .. - } - | QueryOp::SortMergeJoin { - left_input, - right_input, - .. - } => [left_input, right_input] - .into_iter() - .flatten() - .map(|child| &**child) - .collect(), - QueryOp::LateralTopK { outer_plan, .. } | QueryOp::LateralLoop { outer_plan, .. } => { - vec![&**outer_plan] - } - QueryOp::ProviderScan { .. } - | QueryOp::PartialAggregate { .. } - | QueryOp::ShuffleJoinConsume { .. } - | QueryOp::ShuffleAggregateConsume { .. } - | QueryOp::FacetCounts { .. } - | QueryOp::RecursiveScan { .. } - | QueryOp::RecursiveValue { .. } => Vec::new(), - } -} diff --git a/nodedb/src/control/planner/redaction_refusal/expr_reads/search.rs b/nodedb/src/control/planner/redaction_refusal/expr_reads/search.rs index 35d24e997..5aa308e17 100644 --- a/nodedb/src/control/planner/redaction_refusal/expr_reads/search.rs +++ b/nodedb/src/control/planner/redaction_refusal/expr_reads/search.rs @@ -6,7 +6,7 @@ //! predicates a search carries beside its match are read here, and so is the //! vector column a similarity search ranks by. -use nodedb_physical::physical_plan::{PhysicalPlan, TextOp, VectorOp, VectorWriteTargets}; +use nodedb_physical::physical_plan::{TextOp, VectorOp, VectorWriteTargets}; use super::reads::{Clause, NodeReads}; @@ -105,40 +105,3 @@ fn target_reads(targets: &VectorWriteTargets, out: &mut NodeReads) -> crate::Res VectorWriteTargets::Surrogates(_) => Ok(()), } } - -/// The resolved sub-plan a vector search takes its prefilter from, if any. -pub(super) fn vector_child(op: &VectorOp) -> Option<&PhysicalPlan> { - match op { - VectorOp::Search { - inline_prefilter_plan: Some(child), - .. - } => Some(child), - VectorOp::ResolveDirectWrite(inner) => vector_child(inner), - VectorOp::Search { .. } - | VectorOp::Insert { .. } - | VectorOp::BatchInsert { .. } - | VectorOp::MultiSearch { .. } - | VectorOp::Delete { .. } - | VectorOp::DeleteBySurrogate { .. } - | VectorOp::SetParams { .. } - | VectorOp::DropIndex { .. } - | VectorOp::QueryStats { .. } - | VectorOp::Seal { .. } - | VectorOp::CompactIndex { .. } - | VectorOp::Rebuild { .. } - | VectorOp::SparseInsert { .. } - | VectorOp::SparseSearch { .. } - | VectorOp::SparseDelete { .. } - | VectorOp::MultiVectorInsert { .. } - | VectorOp::MultiVectorDelete { .. } - | VectorOp::MultiVectorScoreSearch { .. } - | VectorOp::DirectUpsert { .. } - | VectorOp::DirectInsert { .. } - | VectorOp::DirectInsertIfAbsent { .. } - | VectorOp::DirectBatchWrite { .. } - | VectorOp::DirectDelete { .. } - | VectorOp::DirectTruncate { .. } - | VectorOp::DirectUpdate { .. } - | VectorOp::ResolvedDirectWrite { .. } => None, - } -} diff --git a/nodedb/src/control/planner/redaction_refusal/expr_reads/walk.rs b/nodedb/src/control/planner/redaction_refusal/expr_reads/walk.rs index 4204876f5..26fe7c4cf 100644 --- a/nodedb/src/control/planner/redaction_refusal/expr_reads/walk.rs +++ b/nodedb/src/control/planner/redaction_refusal/expr_reads/walk.rs @@ -22,6 +22,7 @@ use nodedb_physical::physical_plan::PhysicalPlan; use nodedb_physical::physical_task::PhysicalTask; +use crate::control::planner::plan_children::children; use crate::control::security::auth_context::AuthContext; use crate::control::security::catalog::SystemCatalog; use crate::control::security::redaction::RedactionStore; @@ -190,26 +191,6 @@ fn refuse_key_lookup( Ok(()) } -/// The plans whose rows `plan` reads. Exhaustive over [`PhysicalPlan`]. -fn children(plan: &PhysicalPlan) -> Vec<&PhysicalPlan> { - match plan { - PhysicalPlan::Query(op) => query::children(op), - PhysicalPlan::Document(op) => document::child(op).into_iter().collect(), - PhysicalPlan::Vector(op) => search::vector_child(op).into_iter().collect(), - PhysicalPlan::Graph(_) - | PhysicalPlan::Kv(_) - | PhysicalPlan::Text(_) - | PhysicalPlan::Columnar(_) - | PhysicalPlan::Timeseries(_) - | PhysicalPlan::Spatial(_) - | PhysicalPlan::Crdt(_) - | PhysicalPlan::Meta(_) - | PhysicalPlan::Array(_) - | PhysicalPlan::ClusterArray(_) - | PhysicalPlan::ClusterEvent(_) => Vec::new(), - } -} - /// The columns `plan` itself reads. Exhaustive over [`PhysicalPlan`]. /// /// A graph traversal over a redacted collection is refused whole by the diff --git a/nodedb/src/control/planner/sql_plan_convert/array_alter_convert.rs b/nodedb/src/control/planner/sql_plan_convert/array_alter_convert.rs index 56914c57f..e8e141f60 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_alter_convert.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_alter_convert.rs @@ -6,11 +6,14 @@ //! change. The front door proposes the updated entry as a replicated //! `PutArray` (`array_catalog::ddl`). +use nodedb_sql::AlterArrayVisitArgs; + use crate::bridge::envelope::PhysicalPlan; use crate::control::array_catalog::ArrayCatalogEntry; +use crate::control::array_catalog::georef::regeoref_schema; +use crate::control::array_catalog::retention::check_retention_floor; use crate::types::TenantId; use nodedb_physical::physical_plan::MetaOp; -use nodedb_types::config::retention::BitemporalRetention; use super::convert::ConvertContext; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; @@ -21,13 +24,20 @@ use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; /// - `None` = key absent from SET clause → field unchanged. /// - `Some(None)` = key present with value `NULL` → unregister. /// - `Some(Some(v))` = key present with value `v` → update. +/// +/// A georef change re-encodes the schema. Its content bytes stay equal, so +/// `schema_hash`, segments, and the Data Plane open check stay valid. pub(super) fn convert_alter_array( - name: &str, - audit_retain_ms: Option>, - minimum_audit_retain_ms: Option, + args: AlterArrayVisitArgs<'_>, tenant_id: TenantId, ctx: &ConvertContext, ) -> crate::Result> { + let AlterArrayVisitArgs { + name, + audit_retain_ms, + minimum_audit_retain_ms, + georef, + } = args; ctx.require_execute("ALTER ARRAY")?; let array_catalog = ctx .array_catalog @@ -46,25 +56,20 @@ pub(super) fn convert_alter_array( })? }; - // 2. Compute updated fields. - let new_min = - minimum_audit_retain_ms.unwrap_or_else(|| current.minimum_audit_retain_ms.unwrap_or(0)); - let new_retain = match audit_retain_ms { - None => current.audit_retain_ms, - Some(inner) => inner, - }; + // 2. Floor validation before any state mutation. The catalog apply + // repeats it against the committed entry. + check_retention_floor( + name, + ¤t, + audit_retain_ms, + minimum_audit_retain_ms.map(Some), + )?; - // 3. Floor validation before any state mutation. - if let Some(retain_ms) = new_retain { - let retention = BitemporalRetention { - data_retain_ms: 0, - audit_retain_ms: retain_ms as u64, - minimum_audit_retain_ms: new_min, - }; - retention.validate().map_err(|e| crate::Error::PlanError { - detail: format!("ALTER ARRAY {name}: {e}"), - })?; - } + // 3. Re-encode the schema for a georef change. + let statement = format!("ALTER ARRAY {name}"); + let schema_msgpack = georef + .map(|change| regeoref_schema(&statement, ¤t.schema_msgpack, change)) + .transpose()?; let vshard = ctx.collection_key(name).vshard(); Ok(vec![PhysicalTask { @@ -75,6 +80,7 @@ pub(super) fn convert_alter_array( array_id: name.to_string(), audit_retain_ms, minimum_audit_retain_ms: minimum_audit_retain_ms.map(Some), + schema_msgpack, }), post_set_op: PostSetOp::None, txn_id: None, diff --git a/nodedb/src/control/planner/sql_plan_convert/array_convert/ddl.rs b/nodedb/src/control/planner/sql_plan_convert/array_convert/ddl.rs index 8e4f89147..9acd32f4d 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_convert/ddl.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_convert/ddl.rs @@ -13,11 +13,13 @@ use nodedb_array::schema::{ use nodedb_array::types::ArrayId; use nodedb_array::types::domain::{Domain, DomainBound}; use nodedb_sql::types_array::{ - ArrayAttrAst, ArrayCellOrderAst, ArrayDimAst, ArrayDimType, ArrayDomainBound, ArrayTileOrderAst, + ArrayAttrAst, ArrayCellOrderAst, ArrayDimAst, ArrayDimType, ArrayDomainBound, ArrayGeorefAst, + ArrayTileOrderAst, }; use crate::bridge::envelope::PhysicalPlan; use crate::control::array_catalog::ArrayCatalogEntry; +use crate::control::array_catalog::georef::{georef_from_ast, schema_error}; use crate::types::TenantId; use nodedb_physical::physical_plan::ArrayOp; @@ -33,6 +35,7 @@ pub(in super::super) struct CreateArrayArgs<'a> { pub tile_extents: &'a [i64], pub cell_order: ArrayCellOrderAst, pub tile_order: ArrayTileOrderAst, + pub georef: Option<&'a ArrayGeorefAst>, pub prefix_bits: u8, pub audit_retain_ms: Option, pub minimum_audit_retain_ms: Option, @@ -50,6 +53,7 @@ pub(in super::super) fn convert_create_array( tile_extents, cell_order, tile_order, + georef, prefix_bits, audit_retain_ms, minimum_audit_retain_ms, @@ -76,7 +80,15 @@ pub(in super::super) fn convert_create_array( } // 1. Build typed schema. - let schema = build_schema(name, dims, attrs, tile_extents, cell_order, tile_order)?; + let schema = build_schema(SchemaParts { + name, + dims, + attrs, + tile_extents, + cell_order, + tile_order, + georef, + })?; // 2. Encode + hash. let schema_msgpack = @@ -191,25 +203,47 @@ pub(in super::super) fn convert_drop_array( // ── Schema construction helpers ────────────────────────────────────── -pub(super) fn build_schema( - name: &str, - dims: &[ArrayDimAst], - attrs: &[ArrayAttrAst], - tile_extents: &[i64], - cell_order: ArrayCellOrderAst, - tile_order: ArrayTileOrderAst, -) -> crate::Result> { +/// The SQL parts of an array schema. +pub(super) struct SchemaParts<'a> { + pub name: &'a str, + pub dims: &'a [ArrayDimAst], + pub attrs: &'a [ArrayAttrAst], + pub tile_extents: &'a [i64], + pub cell_order: ArrayCellOrderAst, + pub tile_order: ArrayTileOrderAst, + pub georef: Option<&'a ArrayGeorefAst>, +} + +pub(super) fn build_schema(parts: SchemaParts<'_>) -> crate::Result> { + let SchemaParts { + name, + dims, + attrs, + tile_extents, + cell_order, + tile_order, + georef, + } = parts; + let engine_dims: Vec = dims + .iter() + .map(|d| { + let dtype = match d.dtype { + ArrayDimType::Int64 => EngineDimType::Int64, + ArrayDimType::Float64 => EngineDimType::Float64, + ArrayDimType::TimestampMs => EngineDimType::TimestampMs, + ArrayDimType::String => EngineDimType::String, + }; + let lo = bound_to_engine(&d.lo); + let hi = bound_to_engine(&d.hi); + DimSpec::new(d.name.clone(), dtype, Domain::new(lo, hi)) + }) + .collect(); let mut builder = ArraySchemaBuilder::new(name); - for d in dims { - let dtype = match d.dtype { - ArrayDimType::Int64 => EngineDimType::Int64, - ArrayDimType::Float64 => EngineDimType::Float64, - ArrayDimType::TimestampMs => EngineDimType::TimestampMs, - ArrayDimType::String => EngineDimType::String, - }; - let lo = bound_to_engine(&d.lo); - let hi = bound_to_engine(&d.hi); - builder = builder.dim(DimSpec::new(d.name.clone(), dtype, Domain::new(lo, hi))); + if let Some(georef) = georef { + builder = builder.georef(georef_from_ast(name, georef, &engine_dims)?); + } + for d in engine_dims { + builder = builder.dim(d); } for a in attrs { let dtype = super::attr_type::engine_attr_type(a.dtype); @@ -220,9 +254,9 @@ pub(super) fn build_schema( .tile_extents(extents) .cell_order(map_cell_order(cell_order)) .tile_order(map_tile_order(tile_order)); - let schema = builder.build().map_err(|e| crate::Error::PlanError { - detail: format!("CREATE ARRAY {name}: {e}"), - })?; + let schema = builder + .build() + .map_err(|e| schema_error(&format!("CREATE ARRAY {name}"), e))?; Ok(Arc::new(schema)) } diff --git a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/region.rs b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/region.rs index 13fb6dbb8..b9d204944 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/region.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/region.rs @@ -1,17 +1,25 @@ // SPDX-License-Identifier: BUSL-1.1 -//! The region an `ARRAY_AGG` reads, lowered to the engine [`ArrayRegion`]. +//! The region an `ARRAY_AGG` or `ARRAY_SLICE` reads, lowered to the engine +//! [`ArrayRegion`] and [`PolygonZone`]. //! //! The box comes from the slice argument. A zone also narrows the box on its -//! two dims to the bounding box of its polygons, so tile pruning and the -//! cluster fan-out see the zone's extent. An INT64 dim takes the ceiling of -//! the low edge and the floor of the high edge. +//! two dims, so tile pruning and the cluster fan-out see the zone's extent: +//! +//! - A zone in dim coordinates narrows to the bounding box of its polygons. +//! - A zone with an SRID narrows to its frame's dim box. +//! +//! An INT64 dim takes the ceiling of the low edge and the floor of the high +//! edge. A narrowed range can come out empty. The read then returns no cell. use nodedb_array::query::region::{ArrayRegion, PolygonZone, ZonePolygon}; -use nodedb_array::query::slice::DimRange; +use nodedb_array::query::slice::{DimRange, Slice}; +use nodedb_array::query::zone_frame::{ZoneFrameError, build_zone_frame}; use nodedb_array::schema::{ArraySchema, DimType}; use nodedb_array::types::domain::DomainBound; use nodedb_sql::types_array::{ArraySliceAst, ArrayZoneAst}; +use nodedb_types::NodeDbError; +use nodedb_types::geometry::Srid; use super::helpers::resolve_slice; @@ -27,28 +35,55 @@ pub(super) fn resolve_region( return Ok(None); } let mut engine_slice = resolve_slice("ARRAY_AGG", name, slice, schema)?; - let zone = match zone { + let zone = zone + .map(|z| lower_zone("ARRAY_AGG", name, z, schema, &mut engine_slice)) + .transpose()?; + Ok(Some(ArrayRegion::new(engine_slice, zone))) +} + +/// Lower `zone` of `func` over array `name` and narrow `slice` to its extent. +pub(super) fn lower_zone( + func: &str, + name: &str, + zone: &ArrayZoneAst, + schema: &ArraySchema, + slice: &mut Slice, +) -> crate::Result { + let x = zone_axis(func, name, &zone.x_dim, schema)?; + let y = zone_axis(func, name, &zone.y_dim, schema)?; + let polygons = lower_polygons(func, zone)?; + let frame = match zone.srid { None => None, - Some(zone) => { - let x = zone_axis(name, &zone.x_dim, schema)?; - let y = zone_axis(name, &zone.y_dim, schema)?; - let lowered = lower_zone(zone, x.dim, y.dim)?; - if let Some([x0, y0, x1, y1]) = zone_bbox(&lowered.polygons) { - for (axis, lo, hi) in [(x, x0, x1), (y, y0, y1)] { - let (lo, hi) = match axis.kind { - AxisKind::Int64 => ( - DomainBound::Int64(lo.ceil() as i64), - DomainBound::Int64(hi.floor() as i64), - ), - AxisKind::Float64 => (DomainBound::Float64(lo), DomainBound::Float64(hi)), - }; - narrow(&mut engine_slice.dim_ranges[axis.dim as usize], lo, hi); - } - } - Some(lowered) + Some(code) => { + let zone_srid = Srid::try_from(code).map_err(|e| crate::Error::DataException { + detail: format!("{func}: zone SRID: {e}"), + })?; + check_georef_axes(func, name, schema, x.dim, y.dim)?; + Some(build_zone_frame(schema, zone_srid, &polygons).map_err(frame_error)?) } }; - Ok(Some(ArrayRegion::new(engine_slice, zone))) + let extent = match &frame { + Some(frame) => Some(frame.dim_bbox), + None => zone_bbox(&polygons), + }; + if let Some([x0, y0, x1, y1]) = extent { + for (axis, lo, hi) in [(x, x0, x1), (y, y0, y1)] { + let (lo, hi) = match axis.kind { + AxisKind::Int64 => ( + DomainBound::Int64(int_bound(func, lo.ceil())?), + DomainBound::Int64(int_bound(func, hi.floor())?), + ), + AxisKind::Float64 => (DomainBound::Float64(lo), DomainBound::Float64(hi)), + }; + narrow(&mut slice.dim_ranges[axis.dim as usize], lo, hi); + } + } + Ok(PolygonZone { + x_dim: x.dim, + y_dim: y.dim, + polygons, + frame, + }) } /// The numeric type of a zone dim. @@ -65,13 +100,13 @@ struct ZoneAxis { kind: AxisKind, } -fn zone_axis(name: &str, dim: &str, schema: &ArraySchema) -> crate::Result { +fn zone_axis(func: &str, name: &str, dim: &str, schema: &ArraySchema) -> crate::Result { let idx = schema .dims .iter() .position(|d| d.name == dim) .ok_or_else(|| crate::Error::PlanError { - detail: format!("ARRAY_AGG: array '{name}' has no dim '{dim}'"), + detail: format!("{func}: array '{name}' has no dim '{dim}'"), })?; let kind = match schema.dims[idx].dtype { DimType::Int64 => AxisKind::Int64, @@ -79,37 +114,85 @@ fn zone_axis(name: &str, dim: &str, schema: &ArraySchema) -> crate::Result { return Err(crate::Error::PlanError { detail: format!( - "ARRAY_AGG: zone dim '{dim}' is {other:?}; a zone needs an INT64 or FLOAT64 \ - dim" + "{func}: zone dim '{dim}' is {other:?}; a zone needs an INT64 or FLOAT64 dim" ), }); } }; let dim = u32::try_from(idx).map_err(|_| crate::Error::PlanError { - detail: format!("ARRAY_AGG: zone dim '{dim}' is past the dim index range"), + detail: format!("{func}: zone dim '{dim}' is past the dim index range"), })?; Ok(ZoneAxis { dim, kind }) } -fn lower_zone(zone: &ArrayZoneAst, x_dim: u32, y_dim: u32) -> crate::Result { - let polygons = zone - .polygons +/// A zone with an SRID is drawn over the GEOREF axes, x then y. +fn check_georef_axes( + func: &str, + name: &str, + schema: &ArraySchema, + x_dim: u32, + y_dim: u32, +) -> crate::Result<()> { + let prerequisite = |detail: String| crate::Error::ObjectNotInPrerequisiteState { + object: format!("array '{name}'"), + detail, + }; + let georef = schema.georef.as_ref().ok_or_else(|| { + prerequisite(format!( + "{func}: array '{name}' has no GEOREF; a zone with an SRID prefix needs one" + )) + })?; + if (georef.x_dim, georef.y_dim) != (x_dim, y_dim) { + return Err(prerequisite(format!( + "{func}: a zone with an SRID prefix is drawn over the GEOREF axes of array \ + '{name}', x then y" + ))); + } + Ok(()) +} + +fn lower_polygons(func: &str, zone: &ArrayZoneAst) -> crate::Result> { + zone.polygons .iter() .map(|rings| { let (exterior, holes) = rings.split_first().ok_or_else(|| crate::Error::PlanError { - detail: "ARRAY_AGG: a zone polygon has no exterior ring".into(), + detail: format!("{func}: a zone polygon has no exterior ring"), })?; Ok(ZonePolygon { exterior: exterior.clone(), holes: holes.to_vec(), }) }) - .collect::>>()?; - Ok(PolygonZone { - x_dim, - y_dim, - polygons, - }) + .collect() +} + +/// The SQLSTATE each frame refusal reports. +fn frame_error(e: ZoneFrameError) -> crate::Error { + match e { + ZoneFrameError::NoGeoref { ref array } => crate::Error::ObjectNotInPrerequisiteState { + object: format!("array '{array}'"), + detail: e.to_string(), + }, + ZoneFrameError::VertexBudget { .. } => { + crate::Error::from(NodeDbError::program_limit_exceeded(e.to_string())) + } + ZoneFrameError::OutOfDomain { .. } | ZoneFrameError::EmptyZone { .. } => { + crate::Error::DataException { + detail: e.to_string(), + } + } + } +} + +/// A rounded zone edge as an INT64 bound. A bound past `i64` clamps to it. +fn int_bound(func: &str, v: f64) -> crate::Result { + if !v.is_finite() { + return Err(crate::Error::DataException { + detail: format!("{func}: zone extent {v} is not finite"), + }); + } + // `as` saturates a finite value at the `i64` limits. + Ok(v as i64) } /// `[min_x, min_y, max_x, max_y]` over every exterior ring, or `None` when @@ -141,7 +224,9 @@ fn narrow(range: &mut Option, lo: DomainBound, hi: DomainBound) { #[cfg(test)] mod tests { use super::*; - use nodedb_array::schema::{ArraySchemaBuilder, AttrSpec, AttrType, DimSpec}; + use nodedb_array::schema::{ + ArrayGeoref, ArraySchemaBuilder, AttrSpec, AttrType, DimSpec, GeoTransform, + }; use nodedb_array::types::domain::Domain; use nodedb_sql::types_array::{ArrayCoordLiteral, NamedDimRange}; @@ -168,6 +253,31 @@ mod tests { .expect("schema build") } + /// A 16x16 pixel raster of 1-degree WGS 84 pixels from (100°E, 8°N). + fn pixel_schema() -> ArraySchema { + let dim = |name: &str| { + DimSpec::new( + name.to_string(), + DimType::Int64, + Domain::new(DomainBound::Int64(0), DomainBound::Int64(15)), + ) + }; + ArraySchemaBuilder::new("img") + .dim(dim("row")) + .dim(dim("col")) + .attr(AttrSpec::new("b1".to_string(), AttrType::Float64, true)) + .tile_extents(vec![4, 4]) + .build() + .expect("schema build") + .with_georef(Some(ArrayGeoref { + srid: Srid::Wgs84, + x_dim: 1, + y_dim: 0, + transform: Some(GeoTransform([100.0, 1.0, 0.0, 8.0, 0.0, -1.0])), + })) + .expect("georef") + } + fn zone(x: &str, y: &str) -> ArrayZoneAst { ArrayZoneAst { polygons: vec![vec![vec![ @@ -179,6 +289,16 @@ mod tests { ]]], x_dim: x.into(), y_dim: y.into(), + srid: None, + } + } + + fn lon_lat_zone(x0: f64, y0: f64, x1: f64, y1: f64) -> ArrayZoneAst { + ArrayZoneAst { + polygons: vec![vec![vec![[x0, y0], [x1, y0], [x1, y1], [x0, y1], [x0, y0]]]], + x_dim: "col".into(), + y_dim: "row".into(), + srid: Some(4326), } } @@ -217,6 +337,7 @@ mod tests { let zone = region.zone.expect("zone"); assert_eq!((zone.x_dim, zone.y_dim), (0, 1)); assert_eq!(zone.polygons.len(), 1); + assert!(zone.frame.is_none()); } #[test] @@ -266,4 +387,106 @@ mod tests { "{err:?}" ); } + + #[test] + fn an_srid_zone_narrows_the_box_to_its_frame() { + let region = resolve_region( + "img", + &ArraySliceAst::default(), + Some(&lon_lat_zone(101.0, 5.0, 103.0, 7.0)), + &pixel_schema(), + ) + .unwrap() + .expect("region"); + let zone = region.zone.as_ref().expect("zone"); + let frame = zone.frame.as_ref().expect("frame"); + assert_eq!( + (frame.array_srid, frame.zone_srid), + (Srid::Wgs84, Srid::Wgs84) + ); + // Cells 1..=2 on both axes hold the zone. The box pads by half a + // pixel and rounds outward: 0..=3. + assert_eq!( + range(®ion, 1), + Some((DomainBound::Int64(0), DomainBound::Int64(3))) + ); + assert_eq!( + range(®ion, 0), + Some((DomainBound::Int64(0), DomainBound::Int64(3))) + ); + } + + #[test] + fn an_srid_zone_beside_the_raster_narrows_to_an_empty_range() { + let mut slice = Slice::new(vec![None, None]); + slice.dim_ranges[1] = Some(DimRange::new(DomainBound::Int64(0), DomainBound::Int64(15))); + let mut engine = slice.clone(); + lower_zone( + "ARRAY_SLICE", + "img", + &lon_lat_zone(130.0, 5.0, 131.0, 6.0), + &pixel_schema(), + &mut engine, + ) + .expect("lowered"); + let col = engine.dim_ranges[1].as_ref().expect("col range"); + assert!(col.lo > col.hi, "an empty range, not an error: {col:?}"); + } + + #[test] + fn an_srid_zone_on_an_array_without_georef_is_object_not_in_prerequisite_state() { + let mut no_georef = pixel_schema(); + no_georef.georef = None; + let err = resolve_region( + "img", + &ArraySliceAst::default(), + Some(&lon_lat_zone(101.0, 5.0, 103.0, 7.0)), + &no_georef, + ) + .expect_err("refused"); + assert!( + matches!(err, crate::Error::ObjectNotInPrerequisiteState { .. }), + "{err:?}" + ); + } + + #[test] + fn an_srid_zone_over_other_dims_is_object_not_in_prerequisite_state() { + let mut swapped = lon_lat_zone(101.0, 5.0, 103.0, 7.0); + std::mem::swap(&mut swapped.x_dim, &mut swapped.y_dim); + let err = resolve_region( + "img", + &ArraySliceAst::default(), + Some(&swapped), + &pixel_schema(), + ) + .expect_err("refused"); + assert!( + matches!(err, crate::Error::ObjectNotInPrerequisiteState { .. }), + "{err:?}" + ); + } + + #[test] + fn frame_refusals_carry_their_sqlstate_class() { + use nodedb_types::error::sqlstate; + let budget = frame_error(ZoneFrameError::VertexBudget { + array: "img".into(), + limit: 8, + }); + let (_, code, _) = + crate::control::server::pgwire::types::error_map::error_to_sqlstate(&budget); + assert_eq!(code, sqlstate::PROGRAM_LIMIT_EXCEEDED); + let domain = frame_error(ZoneFrameError::OutOfDomain { + array: "img".into(), + x: 150.0, + y: 6.0, + zone_srid: Srid::Wgs84, + detail: "too far from the central meridian".into(), + }); + assert!( + matches!(domain, crate::Error::DataException { .. }), + "{domain:?}" + ); + } } diff --git a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs index 89ff79abc..c95f31683 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs @@ -3,8 +3,7 @@ //! ARRAY_SLICE → PhysicalPlan::Array(ArrayOp::Slice) or ClusterArray(Slice). use nodedb_array::types::ArrayId; -use nodedb_sql::temporal::TemporalScope; -use nodedb_sql::types_array::ArraySliceAst; +use nodedb_sql::ArraySliceVisitArgs; use crate::bridge::envelope::PhysicalPlan; use crate::types::TenantId; @@ -12,22 +11,39 @@ use nodedb_physical::physical_plan::{ArrayOp, ClusterArrayOp}; use super::super::convert::ConvertContext; use super::helpers::{decode_schema, load_entry, resolve_attr_indices, resolve_slice}; +use super::region::lower_zone; use super::tile_keys::compute_slice_tile_keys; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; pub(crate) fn convert_slice( - name: &str, - slice_ast: &ArraySliceAst, - attr_projection: &[String], - limit: u32, - temporal: TemporalScope, + args: ArraySliceVisitArgs<'_>, tenant_id: TenantId, ctx: &ConvertContext, ) -> crate::Result> { + let ArraySliceVisitArgs { + name, + slice: slice_ast, + attr_projection, + limit, + zone, + temporal, + } = args; let entry = load_entry(name, tenant_id, ctx)?; let schema = decode_schema(&entry)?; - let slice = resolve_slice("ARRAY_SLICE", name, slice_ast, &schema)?; + let mut slice = resolve_slice("ARRAY_SLICE", name, slice_ast, &schema)?; + // A zone narrows the box before it is encoded, so pruning and the + // cluster fan-out both see the zone's extent. + let zone_msgpack = match zone { + None => Vec::new(), + Some(zone) => { + let lowered = lower_zone("ARRAY_SLICE", name, zone, &schema, &mut slice)?; + zerompk::to_msgpack_vec(&lowered).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("array slice zone encode: {e}"), + })? + } + }; let slice_msgpack = zerompk::to_msgpack_vec(&slice).map_err(|e| crate::Error::Serialization { format: "msgpack".into(), @@ -35,7 +51,7 @@ pub(crate) fn convert_slice( })?; let (system_time, valid_at_ms) = - super::helpers::resolve_array_temporal_scope(temporal, "ARRAY_SLICE")?; + super::helpers::resolve_array_temporal_scope(*temporal, "ARRAY_SLICE")?; let attr_indices = resolve_attr_indices(name, attr_projection, &schema)?; let aid = ArrayId::in_database(tenant_id, ctx.database_id, name); let vshard = ctx.collection_key(name).vshard(); @@ -58,6 +74,7 @@ pub(crate) fn convert_slice( prefix_bits: entry.prefix_bits, system_time, valid_at_ms, + zone_msgpack, }) } else { PhysicalPlan::Array(ArrayOp::Slice { @@ -69,6 +86,7 @@ pub(crate) fn convert_slice( shard_bucket: None, system_time, valid_at_ms, + zone_msgpack, }) }; @@ -90,6 +108,8 @@ mod tests { ArraySchema, ArraySchemaBuilder, AttrSpec, AttrType, DimSpec, DimType, }; use nodedb_array::types::domain::{Domain, DomainBound}; + use nodedb_sql::temporal::TemporalScope; + use nodedb_sql::types_array::ArraySliceAst; fn make_schema() -> (Vec, ArraySchema) { let schema = ArraySchemaBuilder::new("test_arr") @@ -155,20 +175,27 @@ mod tests { (ctx, handle) } - #[test] - fn single_node_emits_local_array() { - let (ctx, _handle) = make_ctx(false); + fn whole_slice(ctx: &ConvertContext) -> Vec { let slice_ast = ArraySliceAst { dim_ranges: vec![] }; - let tasks = convert_slice( - "test_arr", - &slice_ast, - &[], - 100, - TemporalScope::default(), + convert_slice( + ArraySliceVisitArgs { + name: "test_arr", + slice: &slice_ast, + attr_projection: &[], + limit: 100, + zone: None, + temporal: &TemporalScope::default(), + }, TenantId::new(1), - &ctx, + ctx, ) - .expect("convert"); + .expect("convert") + } + + #[test] + fn single_node_emits_local_array() { + let (ctx, _handle) = make_ctx(false); + let tasks = whole_slice(&ctx); assert_eq!(tasks.len(), 1); assert!( matches!(&tasks[0].plan, PhysicalPlan::Array(ArrayOp::Slice { .. })), @@ -180,17 +207,7 @@ mod tests { #[test] fn cluster_mode_emits_cluster_array() { let (ctx, _handle) = make_ctx(true); - let slice_ast = ArraySliceAst { dim_ranges: vec![] }; - let tasks = convert_slice( - "test_arr", - &slice_ast, - &[], - 100, - TemporalScope::default(), - TenantId::new(1), - &ctx, - ) - .expect("convert"); + let tasks = whole_slice(&ctx); assert_eq!(tasks.len(), 1); assert!( matches!( diff --git a/nodedb/src/control/planner/sql_plan_convert/visitor/arms_array.rs b/nodedb/src/control/planner/sql_plan_convert/visitor/arms_array.rs index 090ee1be2..2eb0bb4f1 100644 --- a/nodedb/src/control/planner/sql_plan_convert/visitor/arms_array.rs +++ b/nodedb/src/control/planner/sql_plan_convert/visitor/arms_array.rs @@ -15,6 +15,7 @@ macro_rules! impl_array_arms_for_convert_visitor { tile_extents, cell_order, tile_order, + georef, prefix_bits, audit_retain_ms, minimum_audit_retain_ms, @@ -27,6 +28,7 @@ macro_rules! impl_array_arms_for_convert_visitor { tile_extents, cell_order, tile_order, + georef, prefix_bits, audit_retain_ms, minimum_audit_retain_ms, @@ -51,17 +53,9 @@ macro_rules! impl_array_arms_for_convert_visitor { fn alter_array( &mut self, - name: &str, - audit_retain_ms: Option>, - minimum_audit_retain_ms: Option, + args: nodedb_sql::AlterArrayVisitArgs<'_>, ) -> crate::Result> { - super::super::array_alter_convert::convert_alter_array( - name, - audit_retain_ms, - minimum_audit_retain_ms, - self.tenant_id, - self.ctx, - ) + super::super::array_alter_convert::convert_alter_array(args, self.tenant_id, self.ctx) } fn insert_array( @@ -87,21 +81,9 @@ macro_rules! impl_array_arms_for_convert_visitor { fn array_slice( &mut self, - name: &str, - slice: &nodedb_sql::types_array::ArraySliceAst, - attr_projection: &[String], - limit: u32, - temporal: &nodedb_sql::temporal::TemporalScope, + args: nodedb_sql::ArraySliceVisitArgs<'_>, ) -> crate::Result> { - super::super::array_fn_convert::convert_slice( - name, - slice, - attr_projection, - limit, - *temporal, - self.tenant_id, - self.ctx, - ) + super::super::array_fn_convert::convert_slice(args, self.tenant_id, self.ctx) } fn array_agg( diff --git a/nodedb/src/control/server/dispatch_utils/change_events/extract.rs b/nodedb/src/control/server/dispatch_utils/change_events/extract.rs index 203f5d1da..976600ac9 100644 --- a/nodedb/src/control/server/dispatch_utils/change_events/extract.rs +++ b/nodedb/src/control/server/dispatch_utils/change_events/extract.rs @@ -585,6 +585,7 @@ mod tests { prefix_bits: 8, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, + zone_msgpack: Vec::new(), }); assert!(extract_write_metadata(&slice, TenantId::new(1)).is_empty()); diff --git a/nodedb/src/control/server/exchange/cluster_leaf.rs b/nodedb/src/control/server/exchange/cluster_leaf.rs index 1e90c5eec..0135c2838 100644 --- a/nodedb/src/control/server/exchange/cluster_leaf.rs +++ b/nodedb/src/control/server/exchange/cluster_leaf.rs @@ -245,6 +245,7 @@ mod tests { prefix_bits: 0, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, + zone_msgpack: Vec::new(), } } diff --git a/nodedb/src/control/server/pgwire/handler/routing/catalog.rs b/nodedb/src/control/server/pgwire/handler/routing/catalog.rs index 603a0eeb7..8bd6c1a02 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/catalog.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/catalog.rs @@ -15,8 +15,8 @@ use nodedb_types::DatabaseId; /// /// Returns `None` if the descriptor has been dropped, the /// catalog is unavailable, or the descriptor kind is not -/// currently tracked (only `Collection` goes through this -/// path today; other kinds do not land in the plan cache). +/// tracked. Planning records `Collection` and `Array` +/// descriptors; other kinds do not land in the plan cache. pub(super) fn current_descriptor_version( state: &SharedState, tenant_id: u64, @@ -34,6 +34,16 @@ pub(super) fn current_descriptor_version( .flatten() .filter(|c| c.is_active) .map(|c| c.descriptor_version.max(1)), + nodedb_cluster::DescriptorKind::Array => state + .array_catalog + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .lookup_by_name_in_database( + nodedb_types::TenantId::new(tenant_id), + database_id, + &id.name, + ) + .map(|entry| entry.catalog_version()), _ => None, } } diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/decode.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/decode.rs index 3ecc9572a..13026540d 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/decode.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/decode.rs @@ -368,7 +368,7 @@ fn coord_text(coord: &ArrayCoordLiteral) -> String { } } -fn bound_text(bound: &ArrayDomainBound) -> String { +pub(super) fn bound_text(bound: &ArrayDomainBound) -> String { match bound { ArrayDomainBound::Int64(v) | ArrayDomainBound::TimestampMs(v) => v.to_string(), ArrayDomainBound::Float64(v) => v.to_string(), @@ -384,6 +384,7 @@ mod tests { fn view() -> ArrayCatalogView { ArrayCatalogView { name: "scene".into(), + georef: None, dims: vec![ArrayDimAst { name: "y".into(), dtype: ArrayDimType::Int64, @@ -467,6 +468,7 @@ mod tests { }; ArrayCatalogView { name: "scene".into(), + georef: None, dims: vec![ArrayDimAst { name: "y".into(), dtype: ArrayDimType::Int64, diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/loader.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/loader.rs index ce7e1e240..b6eec6c8c 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/loader.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/loader.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: BUSL-1.1 //! The shared core of an array bulk load. pgwire `COPY FROM STDIN`, -//! `COPY FROM ''` and the native `ArrayPutBatch` opcode are -//! thin transports over it. +//! `COPY FROM ''` (CSV or GeoTIFF) and the native +//! `ArrayPutBatch` opcode are thin transports over it. //! //! A load streams. Each slice of input decodes and checks row by row, and //! each full chunk dispatches through the write funnel and is awaited @@ -42,6 +42,8 @@ pub(crate) enum LoadInput { Text(TextFormat), /// Native request columns, fed with [`ArrayLoader::feed_cells`]. Cells, + /// Decoded GeoTIFF pixels, fed with [`ArrayLoader::feed_rows`]. + Raster, } /// One open array bulk load. Errors carry their SQLSTATE: `22P02` for a @@ -109,7 +111,7 @@ impl ArrayLoader { .bulk_load_route(array, tenant_id, ctx.database_id) .map_err(|e| bulk_load_refusal(e, tenant_id))? { - BulkLoadRoute::ArrayCells(view) => view, + BulkLoadRoute::ArrayCells(view) => *view, BulkLoadRoute::RowInserts => { return Err(DdlError::new( "42P01", @@ -134,6 +136,7 @@ impl ArrayLoader { (format.label(), Some(stream)) } LoadInput::Cells => ("batch", None), + LoadInput::Raster => ("GeoTIFF", None), }; Ok(Self { target: LoadTarget { @@ -219,6 +222,37 @@ impl ArrayLoader { progress.take_source(ctx, target, &mut source).await } + /// Load checked rows in input order. Each item is a row and its 1-based + /// number, or the refusal of that row. + pub(super) async fn feed_rows( + &mut self, + ctx: &ArrayLoadCtx<'_>, + rows: I, + ) -> Result<(), DdlError> + where + I: Iterator>, + { + let Self { + target, progress, .. + } = self; + for next in rows { + let (row_no, row) = + next.map_err(|e| refuse_row(e, target.label, progress.committed))?; + progress.take_row(ctx, target, row_no, row).await?; + } + Ok(()) + } + + /// The array the load writes. + pub(crate) fn view(&self) -> &ArrayCatalogView { + &self.target.view + } + + /// `e` as this load's refusal, naming its input and committed row count. + pub(super) fn refuse(&self, e: RowError) -> DdlError { + refuse_row(e, self.target.label, self.progress.committed) + } + /// Load the stream's last record and every chunk still pending. Returns /// the number of rows loaded. pub(crate) async fn finish(mut self, ctx: &ArrayLoadCtx<'_>) -> Result { diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/mod.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/mod.rs index e7ffebb3b..cee619b99 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/mod.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/mod.rs @@ -4,6 +4,7 @@ pub mod chunker; pub mod decode; pub mod dispatch; pub mod loader; +pub mod raster; pub mod records; pub mod router; pub mod source; diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/bands.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/bands.rs new file mode 100644 index 000000000..4d71dfe85 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/bands.rs @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Which array attr each GeoTIFF band loads into, and the type rule. +//! +//! `BANDS (a, SKIP, b)` maps file bands in order: band 0 to `a`, band 1 +//! nowhere, band 2 to `b`. Without `BANDS`, band `i` loads into attr `i`. +//! A band loads only into an attr that holds every value of its sample type +//! exactly. An attr no band loads into must be nullable, and every cell +//! holds NULL there. + +use nodedb_raster::SampleType; +use nodedb_sql::catalog::ArrayCatalogView; +use nodedb_sql::ddl_ast::statement::CopyBand; +use nodedb_sql::types_array::ArrayAttrType; + +use super::super::decode::RowError; + +/// Where one attr's value comes from. +#[derive(Debug, Clone, PartialEq)] +pub(super) enum AttrSource { + /// No band loads into the attr. It is nullable. + Null, + /// File band `band` loads into the attr. + Band { + band: usize, + /// Whether the attr holds floats, so integer samples widen to `f64`. + float: bool, + nullable: bool, + name: String, + }, +} + +/// The attr sources in schema order, and the file bands that load. +#[derive(Debug)] +pub(super) struct BandMap { + pub attrs: Vec, + pub loaded: Vec, +} + +/// Map `band_count` file bands of `sample` onto the attrs of `view`. +pub(super) fn map_bands( + view: &ArrayCatalogView, + band_count: u16, + sample: SampleType, + bands: Option<&[CopyBand]>, +) -> Result { + let targets = band_targets(view, band_count, bands)?; + let mut attrs = vec![AttrSource::Null; view.attrs.len()]; + let mut loaded = Vec::with_capacity(targets.len()); + for (band, target) in targets.into_iter().enumerate() { + let Some(index) = target else { continue }; + let Some(attr) = view.attrs.get(index) else { + continue; + }; + if !widens(sample, attr.dtype) { + return Err(RowError::new( + 0, + "42804", + format!( + "band {} holds {sample} samples, which attr '{}' ({}) cannot hold exactly; \ + declare the attr as a type that widens {sample}", + band + 1, + attr.name, + attr.dtype.sql_name() + ), + )); + } + if let Some(slot) = attrs.get_mut(index) { + *slot = AttrSource::Band { + band, + float: attr.dtype.is_float(), + nullable: attr.nullable, + name: attr.name.clone(), + }; + } + loaded.push(band); + } + if loaded.is_empty() { + return Err(RowError::new( + 0, + "22023", + "no file band loads into an attr".to_string(), + )); + } + for (attr, source) in view.attrs.iter().zip(&attrs) { + if *source == AttrSource::Null && !attr.nullable { + return Err(RowError::new( + 0, + "23502", + format!( + "attr '{}' is NOT NULL, and no file band loads into it; map a band to it \ + with BANDS (...)", + attr.name + ), + )); + } + } + Ok(BandMap { attrs, loaded }) +} + +/// The attr index each file band loads into, `None` for a skipped band. +fn band_targets( + view: &ArrayCatalogView, + band_count: u16, + bands: Option<&[CopyBand]>, +) -> Result>, RowError> { + let count = usize::from(band_count); + let Some(bands) = bands else { + if count > view.attrs.len() { + return Err(RowError::new( + 0, + "22023", + format!( + "the file has {count} bands, array '{}' has {} attrs; \ + name each band's attr or SKIP with BANDS (...)", + view.name, + view.attrs.len() + ), + )); + } + return Ok((0..count).map(Some).collect()); + }; + if bands.len() != count { + return Err(RowError::new( + 0, + "22023", + format!( + "BANDS lists {} entries, the file has {count} bands; list one entry per band", + bands.len() + ), + )); + } + bands + .iter() + .map(|band| match band { + CopyBand::Skip => Ok(None), + CopyBand::Attr(name) => view + .attrs + .iter() + .position(|attr| attr.name.eq_ignore_ascii_case(name)) + .map(Some) + .ok_or_else(|| { + RowError::new( + 0, + "42703", + format!( + "BANDS names '{name}', which array '{}' has no attr", + view.name + ), + ) + }), + }) + .collect() +} + +/// Whether every `sample` value converts to `attr` without loss. +fn widens(sample: SampleType, attr: ArrayAttrType) -> bool { + use ArrayAttrType as A; + match sample { + SampleType::U8 => matches!( + attr, + A::UInt8 + | A::Int16 + | A::UInt16 + | A::Int32 + | A::UInt32 + | A::Int64 + | A::Float32 + | A::Float64 + ), + SampleType::U16 => matches!( + attr, + A::UInt16 | A::Int32 | A::UInt32 | A::Int64 | A::Float32 | A::Float64 + ), + SampleType::I16 => matches!( + attr, + A::Int16 | A::Int32 | A::Int64 | A::Float32 | A::Float64 + ), + SampleType::U32 => matches!(attr, A::UInt32 | A::Int64 | A::Float64), + SampleType::F32 => matches!(attr, A::Float32 | A::Float64), + SampleType::F64 => matches!(attr, A::Float64), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_sql::types_array::ArrayAttrAst; + + fn view(attrs: &[(&str, ArrayAttrType, bool)]) -> ArrayCatalogView { + ArrayCatalogView { + name: "scene".into(), + dims: Vec::new(), + attrs: attrs + .iter() + .map(|(name, dtype, nullable)| ArrayAttrAst { + name: (*name).into(), + dtype: *dtype, + nullable: *nullable, + }) + .collect(), + tile_extents: Vec::new(), + georef: None, + } + } + + #[test] + fn widening_is_lossless_only() { + use ArrayAttrType as A; + assert!(widens(SampleType::U8, A::UInt8)); + assert!(widens(SampleType::U16, A::Float32)); + assert!(!widens(SampleType::U16, A::Int16)); + assert!(!widens(SampleType::I16, A::UInt16)); + assert!(!widens(SampleType::U32, A::Float32)); + assert!(!widens(SampleType::F32, A::Int64)); + assert!(!widens(SampleType::F64, A::Float32)); + assert!(!widens(SampleType::U8, A::String)); + } + + #[test] + fn bands_load_in_order_without_a_list() { + let v = view(&[ + ("b1", ArrayAttrType::UInt16, false), + ("b2", ArrayAttrType::Float64, true), + ("extra", ArrayAttrType::Int64, true), + ]); + let map = map_bands(&v, 2, SampleType::U16, None).expect("maps"); + assert_eq!(map.loaded, vec![0, 1]); + assert!(matches!( + map.attrs[1], + AttrSource::Band { + band: 1, + float: true, + .. + } + )); + assert_eq!(map.attrs[2], AttrSource::Null); + } + + #[test] + fn a_band_list_skips_and_reorders() { + let v = view(&[ + ("red", ArrayAttrType::UInt16, true), + ("nir", ArrayAttrType::UInt16, true), + ]); + let list = [ + CopyBand::Attr("nir".into()), + CopyBand::Skip, + CopyBand::Attr("red".into()), + ]; + let map = map_bands(&v, 3, SampleType::U16, Some(&list)).expect("maps"); + assert_eq!(map.loaded, vec![0, 2]); + assert!(matches!(map.attrs[0], AttrSource::Band { band: 2, .. })); + assert!(matches!(map.attrs[1], AttrSource::Band { band: 0, .. })); + } + + #[test] + fn refusals_carry_their_sqlstate() { + let narrow = view(&[("b1", ArrayAttrType::UInt8, true)]); + let err = map_bands(&narrow, 1, SampleType::U16, None).expect_err("u16 into UINT8"); + assert_eq!(err.sqlstate, "42804"); + + let strict = view(&[ + ("b1", ArrayAttrType::UInt16, true), + ("qa", ArrayAttrType::UInt16, false), + ]); + let err = map_bands(&strict, 1, SampleType::U16, None).expect_err("qa unmapped"); + assert_eq!(err.sqlstate, "23502"); + + let one = view(&[("b1", ArrayAttrType::UInt16, true)]); + let err = map_bands(&one, 2, SampleType::U16, None).expect_err("2 bands, 1 attr"); + assert_eq!(err.sqlstate, "22023"); + let short = [CopyBand::Attr("b1".into())]; + let err = map_bands(&one, 2, SampleType::U16, Some(&short)).expect_err("short list"); + assert_eq!(err.sqlstate, "22023"); + let unknown = [CopyBand::Attr("b9".into())]; + let err = map_bands(&one, 1, SampleType::U16, Some(&unknown)).expect_err("no attr b9"); + assert_eq!(err.sqlstate, "42703"); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/grid.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/grid.rs new file mode 100644 index 000000000..f650a8799 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/grid.rs @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Where each GeoTIFF pixel lands in an array: the cell coordinates of its +//! GEOREF axes, checked against the dim domains once at open. +//! +//! A pixel-mode array stores pixel indexes. The file grid must be the +//! array grid shifted by whole pixels, so file pixel `(col, row)` is cell +//! `(col + col0, row + row0)`. A world-mode array stores pixel-centre +//! world coordinates, so the cell is the file geotransform at +//! `(col + 0.5, row + 0.5)`. +//! +//! The pixel-to-cell map is affine and each coordinate is monotone in the +//! pixel index. So the four corner pixels bound every cell, and a footprint +//! inside the dim domains puts every cell inside them. + +use nodedb_array::schema::GeoTransform; +use nodedb_sql::catalog::ArrayCatalogView; +use nodedb_sql::types_array::{ArrayCoordLiteral, ArrayDimAst, ArrayDimType, ArrayDomainBound}; + +use super::super::decode::{RowError, bound_text}; + +/// Largest relative difference between the file and array linear terms. +const LINEAR_TOLERANCE: f64 = 1e-9; +/// Largest distance from a whole number, in pixels or INT64 units. +const INTEGRAL_TOLERANCE: f64 = 1e-6; +/// Largest pixel offset kept as `i64`, so `offset + u32` never overflows. +const MAX_OFFSET: f64 = (1u64 << 62) as f64; + +/// The cell coordinates of one pixel on the array's GEOREF axes. +#[derive(Debug, Clone, Copy)] +pub(super) enum CellGrid { + /// File pixel `(col, row)` is cell `(col + col0, row + row0)`. + Pixel { col0: i64, row0: i64 }, + /// The cell is the pixel centre in world coordinates. An `INT64` axis + /// holds the rounded centre, which open checks to be whole. + World { + gt: GeoTransform, + x_int: bool, + y_int: bool, + }, +} + +impl CellGrid { + /// The `(x, y)` cell coordinates of file pixel `(col, row)`. + pub(super) fn cell(&self, col: u32, row: u32) -> (ArrayCoordLiteral, ArrayCoordLiteral) { + match *self { + CellGrid::Pixel { col0, row0 } => ( + ArrayCoordLiteral::Int64(i64::from(col) + col0), + ArrayCoordLiteral::Int64(i64::from(row) + row0), + ), + CellGrid::World { gt, x_int, y_int } => { + let (x, y) = gt.pixel_to_world(f64::from(col) + 0.5, f64::from(row) + 0.5); + (world_coord(x, x_int), world_coord(y, y_int)) + } + } + } +} + +fn world_coord(value: f64, int: bool) -> ArrayCoordLiteral { + if int { + // Open proved every centre on this axis whole and inside the domain. + ArrayCoordLiteral::Int64(value.round() as i64) + } else { + ArrayCoordLiteral::Float64(value) + } +} + +/// The file facts the grid checks read. +pub(super) struct GridInput<'a> { + pub view: &'a ArrayCatalogView, + pub x_dim: &'a ArrayDimAst, + pub y_dim: &'a ArrayDimAst, + /// The array GEOREF transform. `None` is a world-mode array. + pub array_gt: Option<[f64; 6]>, + pub file_gt: [f64; 6], + pub width: u32, + pub height: u32, +} + +/// Check the file grid against the array and its footprint against the +/// dim domains. Misalignment is `22023`, a footprint outside a domain +/// `22003`. +pub(super) fn plan_grid(input: &GridInput<'_>) -> Result { + match input.array_gt { + Some(array_gt) => pixel_grid(input, array_gt), + None => world_grid(input), + } +} + +fn pixel_grid(input: &GridInput<'_>, array_gt: [f64; 6]) -> Result { + let file = input.file_gt; + let scale = [1, 2, 4, 5] + .iter() + .map(|&k| array_gt[k].abs()) + .fold(0.0_f64, f64::max); + let linear_equal = [1, 2, 4, 5] + .iter() + .all(|&k| (file[k] - array_gt[k]).abs() <= LINEAR_TOLERANCE * scale); + if !linear_equal { + return Err(misaligned( + input, + format!( + "the file pixel size and rotation {:?} differ from the array TRANSFORM {:?}", + [file[1], file[2], file[4], file[5]], + [array_gt[1], array_gt[2], array_gt[4], array_gt[5]] + ), + )); + } + let origin = GeoTransform(array_gt).world_to_pixel(file[0], file[3]); + let Some((u0, v0)) = origin else { + return Err(misaligned( + input, + "the array TRANSFORM has no inverse".into(), + )); + }; + let (Some(col0), Some(row0)) = (whole_offset(u0), whole_offset(v0)) else { + return Err(misaligned( + input, + format!("the file origin falls at array pixel ({u0}, {v0}), not on a whole pixel"), + )); + }; + let last_col = col0 + i64::from(input.width) - 1; + let last_row = row0 + i64::from(input.height) - 1; + check_int_span(input.x_dim, col0, last_col)?; + check_int_span(input.y_dim, row0, last_row)?; + Ok(CellGrid::Pixel { col0, row0 }) +} + +/// `value` rounded to `i64` when it is within the tolerance of a whole +/// number and inside the offset range. +fn whole_offset(value: f64) -> Option { + let rounded = value.round(); + let whole = (value - rounded).abs() <= INTEGRAL_TOLERANCE; + (whole && rounded.abs() <= MAX_OFFSET).then_some(rounded as i64) +} + +fn world_grid(input: &GridInput<'_>) -> Result { + let gt = GeoTransform(input.file_gt); + let (w, h) = (f64::from(input.width), f64::from(input.height)); + let corners = [ + (0.5, 0.5), + (w - 0.5, 0.5), + (0.5, h - 0.5), + (w - 0.5, h - 0.5), + ] + .map(|(u, v)| gt.pixel_to_world(u, v)); + let xs = corners.map(|(x, _)| x); + let ys = corners.map(|(_, y)| y); + let x_int = int_axis(input.x_dim)?; + let y_int = int_axis(input.y_dim)?; + let file = input.file_gt; + if x_int { + check_whole_axis(input.x_dim, &xs, [file[1], file[2]])?; + } + if y_int { + check_whole_axis(input.y_dim, &ys, [file[4], file[5]])?; + } + check_world_span(input.x_dim, &xs)?; + check_world_span(input.y_dim, &ys)?; + Ok(CellGrid::World { gt, x_int, y_int }) +} + +/// Whether a world-mode axis is `INT64`. GEOREF admits `INT64` and +/// `FLOAT64` axes only. +fn int_axis(dim: &ArrayDimAst) -> Result { + match dim.dtype { + ArrayDimType::Int64 => Ok(true), + ArrayDimType::Float64 => Ok(false), + ArrayDimType::TimestampMs | ArrayDimType::String => Err(RowError::new( + 0, + "22023", + format!("GEOREF axis '{}' cannot hold world coordinates", dim.name), + )), + } +} + +/// Every pixel centre on an `INT64` world axis must be whole: the corner +/// centres and the per-pixel steps. +fn check_whole_axis( + dim: &ArrayDimAst, + corners: &[f64; 4], + steps: [f64; 2], +) -> Result<(), RowError> { + let whole = |v: &f64| (v - v.round()).abs() <= INTEGRAL_TOLERANCE; + if corners.iter().all(whole) && steps.iter().all(whole) { + return Ok(()); + } + Err(RowError::new( + 0, + "22023", + format!( + "pixel centres on INT64 axis '{}' are not whole numbers; \ + declare it FLOAT64 or resample outside the engine", + dim.name + ), + )) +} + +fn check_world_span(dim: &ArrayDimAst, corners: &[f64; 4]) -> Result<(), RowError> { + let lo = corners.iter().copied().fold(f64::INFINITY, f64::min); + let hi = corners.iter().copied().fold(f64::NEG_INFINITY, f64::max); + match dim.dtype { + ArrayDimType::Int64 => check_int_span(dim, lo.round() as i64, hi.round() as i64), + ArrayDimType::Float64 | ArrayDimType::TimestampMs | ArrayDimType::String => { + let inside = match (&dim.lo, &dim.hi) { + (ArrayDomainBound::Float64(d_lo), ArrayDomainBound::Float64(d_hi)) => { + *d_lo <= lo && hi <= *d_hi + } + _ => false, + }; + if inside { + Ok(()) + } else { + Err(outside(dim, &lo.to_string(), &hi.to_string())) + } + } + } +} + +/// Check the integer span `[lo, hi]` of an axis against its domain. +fn check_int_span(dim: &ArrayDimAst, lo: i64, hi: i64) -> Result<(), RowError> { + let inside = match (&dim.lo, &dim.hi) { + (ArrayDomainBound::Int64(d_lo), ArrayDomainBound::Int64(d_hi)) => { + *d_lo <= lo && hi <= *d_hi + } + _ => false, + }; + if inside { + return Ok(()); + } + Err(outside(dim, &lo.to_string(), &hi.to_string())) +} + +fn outside(dim: &ArrayDimAst, lo: &str, hi: &str) -> RowError { + RowError::new( + 0, + "22003", + format!( + "the raster footprint spans [{lo}..{hi}] on dim '{}', outside its domain [{}..{}]", + dim.name, + bound_text(&dim.lo), + bound_text(&dim.hi) + ), + ) +} + +fn misaligned(input: &GridInput<'_>, why: String) -> RowError { + RowError::new( + 0, + "22023", + format!( + "raster grid does not align with array '{}': {why}; resample outside the engine", + input.view.name + ), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn int_dim(name: &str, lo: i64, hi: i64) -> ArrayDimAst { + ArrayDimAst { + name: name.into(), + dtype: ArrayDimType::Int64, + lo: ArrayDomainBound::Int64(lo), + hi: ArrayDomainBound::Int64(hi), + } + } + + fn float_dim(name: &str, lo: f64, hi: f64) -> ArrayDimAst { + ArrayDimAst { + name: name.into(), + dtype: ArrayDimType::Float64, + lo: ArrayDomainBound::Float64(lo), + hi: ArrayDomainBound::Float64(hi), + } + } + + fn view() -> ArrayCatalogView { + ArrayCatalogView { + name: "scene".into(), + dims: Vec::new(), + attrs: Vec::new(), + tile_extents: Vec::new(), + georef: None, + } + } + + const GT: [f64; 6] = [600_000.0, 30.0, 0.0, 700_000.0, 0.0, -30.0]; + + fn input<'a>( + view: &'a ArrayCatalogView, + x: &'a ArrayDimAst, + y: &'a ArrayDimAst, + array_gt: Option<[f64; 6]>, + file_gt: [f64; 6], + ) -> GridInput<'a> { + GridInput { + view, + x_dim: x, + y_dim: y, + array_gt, + file_gt, + width: 256, + height: 192, + } + } + + #[test] + fn a_shifted_file_lands_at_its_pixel_offset() { + let (v, x, y) = (view(), int_dim("col", 0, 1000), int_dim("row", 0, 1000)); + let file = [ + 600_000.0 + 30.0 * 10.0, + 30.0, + 0.0, + 700_000.0 - 30.0 * 4.0, + 0.0, + -30.0, + ]; + let grid = plan_grid(&input(&v, &x, &y, Some(GT), file)).expect("aligned"); + let (cx, cy) = grid.cell(0, 0); + assert_eq!(cx, ArrayCoordLiteral::Int64(10)); + assert_eq!(cy, ArrayCoordLiteral::Int64(4)); + } + + #[test] + fn a_half_pixel_shift_is_misaligned() { + let (v, x, y) = (view(), int_dim("col", 0, 1000), int_dim("row", 0, 1000)); + let file = [600_015.0, 30.0, 0.0, 700_000.0, 0.0, -30.0]; + let err = plan_grid(&input(&v, &x, &y, Some(GT), file)).expect_err("half pixel"); + assert_eq!(err.sqlstate, "22023"); + assert!( + err.message.contains("resample outside the engine"), + "{}", + err.message + ); + let scaled = [600_000.0, 10.0, 0.0, 700_000.0, 0.0, -10.0]; + let err = plan_grid(&input(&v, &x, &y, Some(GT), scaled)).expect_err("pixel size"); + assert_eq!(err.sqlstate, "22023"); + } + + #[test] + fn a_footprint_past_the_domain_is_22003() { + let (v, x, y) = (view(), int_dim("col", 0, 255), int_dim("row", 0, 99)); + let err = plan_grid(&input(&v, &x, &y, Some(GT), GT)).expect_err("rows past 99"); + assert_eq!(err.sqlstate, "22003"); + } + + #[test] + fn world_cells_are_pixel_centres() { + let v = view(); + let x = float_dim("x", 600_000.0, 607_680.0); + let y = float_dim("y", 694_240.0, 700_000.0); + let grid = plan_grid(&input(&v, &x, &y, None, GT)).expect("inside"); + let (cx, cy) = grid.cell(2, 1); + assert_eq!(cx, ArrayCoordLiteral::Float64(600_075.0)); + assert_eq!(cy, ArrayCoordLiteral::Float64(699_955.0)); + } + + #[test] + fn int_world_axes_need_whole_centres() { + let v = view(); + let x = int_dim("x", 0, 10_000_000); + let y = int_dim("y", 0, 10_000_000); + let quarter = [600_000.0, 25.0, 0.0, 700_000.0, 0.0, -25.0]; + let err = plan_grid(&input(&v, &x, &y, None, quarter)).expect_err("centres at 12.5 m"); + assert_eq!(err.sqlstate, "22023"); + let shifted = [599_985.0, 30.0, 0.0, 700_015.0, 0.0, -30.0]; + let grid = plan_grid(&input(&v, &x, &y, None, shifted)).expect("whole centres"); + let (cx, cy) = grid.cell(1, 1); + assert_eq!(cx, ArrayCoordLiteral::Int64(600_030)); + assert_eq!(cy, ArrayCoordLiteral::Int64(699_970)); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/mod.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/mod.rs new file mode 100644 index 000000000..1a0d8b59e --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/mod.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: BUSL-1.1 + +pub mod bands; +pub mod grid; +pub mod open; +pub mod pixels; +pub mod refusal; +pub mod task; + +pub(crate) use task::copy_raster; diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/open.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/open.rs new file mode 100644 index 000000000..64a401dd8 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/open.rs @@ -0,0 +1,429 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The open-time checks of a GeoTIFF load. Every check runs before the +//! first block decodes, so a refused file commits no row. +//! +//! - The array must have a GEOREF (`55000`), and the file CRS must be its +//! SRID (`22023`). +//! - Bands map onto attrs by the lossless type rule (`42804`), and an attr +//! no band fills must be nullable (`23502`). +//! - The file grid must align with the array (`22023`), and its footprint +//! must fit the dim domains (`22003`). +//! - `TIME` fills the one `TIMESTAMP` dim. Any other dim has no source. +//! - The nodata value must be a value of the sample type (`22023`). + +use nodedb_raster::{BandSamples, SampleType}; +use nodedb_sql::catalog::ArrayCatalogView; +use nodedb_sql::ddl_ast::statement::CopyRasterOptions; +use nodedb_sql::types_array::{ArrayDimAst, ArrayDimType, ArrayDomainBound, ArrayGeorefAst}; +use nodedb_types::geometry::Srid; + +use super::super::decode::{RowError, bound_text}; +use super::bands::{AttrSource, map_bands}; +use super::grid::{CellGrid, GridInput, plan_grid}; + +/// What a GeoTIFF's tags say, read on the decoder thread. +#[derive(Debug, Clone)] +pub(super) struct RasterHeader { + pub width: u32, + pub height: u32, + pub bands: u16, + pub sample_type: SampleType, + /// The pixel-corner geotransform. `None` when the file has none. + pub transform: Option<[f64; 6]>, + /// The CRS the GeoKeys name. `None` when they name none. + pub srid: Option, + /// The `GDAL_NODATA` value. + pub nodata: Option, +} + +/// The source of one dim's coordinate. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(super) enum CoordSource { + /// The GEOREF x axis. + X, + /// The GEOREF y axis. + Y, + /// The `TIME` instant, in ms since the Unix epoch. + Time(i64), +} + +/// The nodata value, typed by the sample type. +#[derive(Debug, Clone, Copy)] +pub(super) enum Nodata { + Int(i64), + F32(f32), + F64(f64), +} + +impl Nodata { + /// Whether sample `index` of `band` is nodata. NaN matches NaN. + pub(super) fn matches(self, band: &BandSamples, index: usize) -> bool { + match (self, band) { + (Nodata::Int(n), BandSamples::U8(v)) => { + v.get(index).is_some_and(|s| i64::from(*s) == n) + } + (Nodata::Int(n), BandSamples::U16(v)) => { + v.get(index).is_some_and(|s| i64::from(*s) == n) + } + (Nodata::Int(n), BandSamples::I16(v)) => { + v.get(index).is_some_and(|s| i64::from(*s) == n) + } + (Nodata::Int(n), BandSamples::U32(v)) => { + v.get(index).is_some_and(|s| i64::from(*s) == n) + } + (Nodata::F32(n), BandSamples::F32(v)) => v + .get(index) + .is_some_and(|s| *s == n || (s.is_nan() && n.is_nan())), + (Nodata::F64(n), BandSamples::F64(v)) => v + .get(index) + .is_some_and(|s| *s == n || (s.is_nan() && n.is_nan())), + ( + Nodata::Int(_) | Nodata::F32(_) | Nodata::F64(_), + BandSamples::U8(_) + | BandSamples::U16(_) + | BandSamples::I16(_) + | BandSamples::U32(_) + | BandSamples::F32(_) + | BandSamples::F64(_), + ) => false, + } + } +} + +/// How every pixel of an accepted file becomes a row. +#[derive(Debug)] +pub(super) struct RasterPlan { + pub(super) width: u32, + pub(super) bands: u16, + /// One source per dim, in schema order. + pub(super) coords: Vec, + /// One source per attr, in schema order. + pub(super) attrs: Vec, + /// The file bands that load. A pixel is nodata when all of them are. + pub(super) loaded: Vec, + pub(super) grid: CellGrid, + pub(super) nodata: Option, +} + +impl RasterPlan { + /// Run every open-time check of `header` against `view`. + pub(super) fn new( + view: &ArrayCatalogView, + georef: &ArrayGeorefAst, + header: &RasterHeader, + options: &CopyRasterOptions, + ) -> Result { + check_srid(view, georef, header)?; + let map = map_bands( + view, + header.bands, + header.sample_type, + options.bands.as_deref(), + )?; + let (x_index, x_dim) = georef_axis(view, &georef.x_dim)?; + let (y_index, y_dim) = georef_axis(view, &georef.y_dim)?; + let file_gt = header.transform.ok_or_else(|| { + RowError::new( + 0, + "22023", + "the file has no geotransform, so no pixel has a world position".to_string(), + ) + })?; + let grid = plan_grid(&GridInput { + view, + x_dim, + y_dim, + array_gt: georef.transform, + file_gt, + width: header.width, + height: header.height, + })?; + let coords = plan_coords(view, x_index, y_index, options.time_ms)?; + let nodata = options + .nodata + .or(header.nodata) + .map(|value| typed_nodata(value, header.sample_type)) + .transpose()?; + Ok(Self { + width: header.width, + bands: header.bands, + coords, + attrs: map.attrs, + loaded: map.loaded, + grid, + nodata, + }) + } +} + +fn check_srid( + view: &ArrayCatalogView, + georef: &ArrayGeorefAst, + header: &RasterHeader, +) -> Result<(), RowError> { + match header.srid { + Some(srid) if srid.code() == georef.srid => Ok(()), + Some(srid) => Err(RowError::new( + 0, + "22023", + format!( + "the file CRS is EPSG:{}, array '{}' GEOREF is SRID {}; \ + reproject outside the engine", + srid.code(), + view.name, + georef.srid + ), + )), + None => Err(RowError::new( + 0, + "22023", + format!( + "the file GeoKeys name no CRS, array '{}' GEOREF is SRID {}", + view.name, georef.srid + ), + )), + } +} + +/// The index and spec of the dim a GEOREF axis names. +fn georef_axis<'v>( + view: &'v ArrayCatalogView, + name: &str, +) -> Result<(usize, &'v ArrayDimAst), RowError> { + view.dims + .iter() + .enumerate() + .find(|(_, dim)| dim.name.eq_ignore_ascii_case(name)) + .ok_or_else(|| { + RowError::new( + 0, + "XX000", + format!( + "array '{}' GEOREF names dim '{name}', which it lacks", + view.name + ), + ) + }) +} + +/// The source of every dim: the GEOREF axes, then `TIME` for the one +/// `TIMESTAMP` dim. Any other dim has no source in a raster (`0A000`). +fn plan_coords( + view: &ArrayCatalogView, + x_index: usize, + y_index: usize, + time_ms: Option, +) -> Result, RowError> { + let mut time_used = false; + let mut coords = Vec::with_capacity(view.dims.len()); + for (index, dim) in view.dims.iter().enumerate() { + let source = if index == x_index { + CoordSource::X + } else if index == y_index { + CoordSource::Y + } else if dim.dtype == ArrayDimType::TimestampMs && !time_used { + time_used = true; + CoordSource::Time(time_coord(dim, time_ms)?) + } else { + return Err(RowError::new( + 0, + "0A000", + format!( + "dim '{}' has no source in a GeoTIFF; a raster fills the GEOREF axes \ + and one TIMESTAMP dim from TIME", + dim.name + ), + )); + }; + coords.push(source); + } + if time_ms.is_some() && !time_used { + return Err(RowError::new( + 0, + "42601", + format!( + "TIME is given, and array '{}' has no TIMESTAMP dim", + view.name + ), + )); + } + Ok(coords) +} + +/// The `TIME` instant for `dim`, checked against its domain. +fn time_coord(dim: &ArrayDimAst, time_ms: Option) -> Result { + let time = time_ms.ok_or_else(|| { + RowError::new( + 0, + "42601", + format!( + "dim '{}' is a TIMESTAMP; give the scene instant with WITH (TIME '')", + dim.name + ), + ) + })?; + let inside = match (&dim.lo, &dim.hi) { + ( + ArrayDomainBound::TimestampMs(lo) | ArrayDomainBound::Int64(lo), + ArrayDomainBound::TimestampMs(hi) | ArrayDomainBound::Int64(hi), + ) => *lo <= time && time <= *hi, + _ => false, + }; + if inside { + return Ok(time); + } + Err(RowError::new( + 0, + "22003", + format!( + "TIME {time} is outside dim '{}' domain [{}..{}]", + dim.name, + bound_text(&dim.lo), + bound_text(&dim.hi) + ), + )) +} + +/// `value` as a nodata of `sample`. An integer type needs a whole value in +/// its range. `f32` takes the nearest `f32`, as GDAL compares. +fn typed_nodata(value: f64, sample: SampleType) -> Result { + let range = match sample { + SampleType::U8 => Some((f64::from(u8::MIN), f64::from(u8::MAX))), + SampleType::U16 => Some((f64::from(u16::MIN), f64::from(u16::MAX))), + SampleType::I16 => Some((f64::from(i16::MIN), f64::from(i16::MAX))), + SampleType::U32 => Some((f64::from(u32::MIN), f64::from(u32::MAX))), + SampleType::F32 if value.is_finite() && value.abs() > f64::from(f32::MAX) => { + return Err(unrepresentable(value, sample)); + } + SampleType::F32 => return Ok(Nodata::F32(value as f32)), + SampleType::F64 => return Ok(Nodata::F64(value)), + }; + match range { + Some((lo, hi)) if value.fract() == 0.0 && lo <= value && value <= hi => { + Ok(Nodata::Int(value as i64)) + } + _ => Err(unrepresentable(value, sample)), + } +} + +fn unrepresentable(value: f64, sample: SampleType) -> RowError { + RowError::new( + 0, + "22023", + format!("nodata {value} is not a {sample} value, so no sample can match it"), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn integer_nodata_must_be_a_sample_value() { + assert!(matches!( + typed_nodata(0.0, SampleType::U16), + Ok(Nodata::Int(0)) + )); + assert!(matches!( + typed_nodata(-9999.0, SampleType::I16), + Ok(Nodata::Int(-9999)) + )); + for (value, sample) in [ + (-1.0, SampleType::U16), + (65536.0, SampleType::U16), + (0.5, SampleType::U8), + (f64::NAN, SampleType::U32), + (1e39, SampleType::F32), + ] { + let err = typed_nodata(value, sample).expect_err("unrepresentable"); + assert_eq!(err.sqlstate, "22023", "{value} as {sample}"); + } + } + + #[test] + fn nan_nodata_matches_nan_samples() { + let band = BandSamples::F32(vec![f32::NAN, 1.0]); + let nodata = typed_nodata(f64::NAN, SampleType::F32).expect("NaN is an f32 value"); + assert!(nodata.matches(&band, 0)); + assert!(!nodata.matches(&band, 1)); + let ints = BandSamples::U16(vec![0, 7]); + assert!(Nodata::Int(0).matches(&ints, 0)); + assert!(!Nodata::Int(0).matches(&ints, 1)); + } + + fn dim( + name: &str, + dtype: ArrayDimType, + lo: ArrayDomainBound, + hi: ArrayDomainBound, + ) -> ArrayDimAst { + ArrayDimAst { + name: name.into(), + dtype, + lo, + hi, + } + } + + fn view(dims: Vec) -> ArrayCatalogView { + ArrayCatalogView { + name: "scene".into(), + dims, + attrs: Vec::new(), + tile_extents: Vec::new(), + georef: None, + } + } + + #[test] + fn time_fills_the_timestamp_dim() { + let ts = || { + dim( + "t", + ArrayDimType::TimestampMs, + ArrayDomainBound::TimestampMs(0), + ArrayDomainBound::TimestampMs(1_000), + ) + }; + let axis = |name: &str| { + dim( + name, + ArrayDimType::Int64, + ArrayDomainBound::Int64(0), + ArrayDomainBound::Int64(9), + ) + }; + let v = view(vec![ts(), axis("row"), axis("col")]); + let coords = plan_coords(&v, 2, 1, Some(500)).expect("time given"); + assert_eq!( + coords, + vec![CoordSource::Time(500), CoordSource::Y, CoordSource::X] + ); + assert_eq!( + plan_coords(&v, 2, 1, None).expect_err("no TIME").sqlstate, + "42601" + ); + assert_eq!( + plan_coords(&v, 2, 1, Some(5_000)) + .expect_err("past") + .sqlstate, + "22003" + ); + + let plain = view(vec![axis("row"), axis("col")]); + assert_eq!( + plan_coords(&plain, 1, 0, Some(5)) + .expect_err("no ts") + .sqlstate, + "42601" + ); + let band = view(vec![axis("band"), axis("row"), axis("col")]); + assert_eq!( + plan_coords(&band, 2, 1, None) + .expect_err("band dim") + .sqlstate, + "0A000" + ); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/pixels.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/pixels.rs new file mode 100644 index 000000000..f4ed8d0cc --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/pixels.rs @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The rows of one decoded GeoTIFF block, read one pixel at a time. +//! +//! A row's number is the pixel's row-major ordinal in the image plus one. +//! A pixel whose loaded bands all hold nodata is skipped. A kept pixel +//! with one nodata band writes NULL there, or is refused with `23502` when +//! the attr is NOT NULL. Open checked the types and the footprint, so no +//! other check runs per pixel. +//! +//! Each kept pixel allocates its coordinate and attr vectors, sized +//! exactly. A skipped pixel allocates nothing. + +use nodedb_raster::{BandSamples, RasterBlock}; +use nodedb_sql::types_array::{ArrayAttrLiteral, ArrayCoordLiteral, ArrayInsertRow}; + +use super::super::decode::RowError; +use super::bands::AttrSource; +use super::open::{CoordSource, RasterPlan}; + +/// The rows of one block in pixel order within the block. +pub(super) struct BlockRows<'a> { + plan: &'a RasterPlan, + block: &'a RasterBlock, + next: usize, + pixels: usize, +} + +impl<'a> BlockRows<'a> { + /// Read `block` under `plan`. A block with the wrong band count or a + /// short band is a malformed file (`22P04`). + pub(super) fn new(plan: &'a RasterPlan, block: &'a RasterBlock) -> Result { + let pixels = block.window.pixels(); + if block.bands.len() != usize::from(plan.bands) { + return Err(RowError::new( + 0, + "22P04", + format!( + "GeoTIFF block {} decoded {} bands, the file declares {}", + block.index, + block.bands.len(), + plan.bands + ), + )); + } + if let Some(short) = block.bands.iter().find(|band| band.len() < pixels) { + return Err(RowError::new( + 0, + "22P04", + format!( + "GeoTIFF block {} decoded {} samples, its window holds {pixels}", + block.index, + short.len() + ), + )); + } + Ok(Self { + plan, + block, + next: 0, + pixels, + }) + } + + /// Whether every loaded band of pixel `index` holds nodata. + fn is_nodata(&self, index: usize) -> bool { + let Some(nodata) = self.plan.nodata else { + return false; + }; + self.plan.loaded.iter().all(|&band| { + self.block + .bands + .get(band) + .is_some_and(|samples| nodata.matches(samples, index)) + }) + } + + /// The row of pixel `index` of the block. + fn row(&self, index: usize) -> Result<(usize, ArrayInsertRow), RowError> { + let window = &self.block.window; + let local_row = index / window.width as usize; + let local_col = index % window.width as usize; + // The window lies inside the image, so both fit `u32`. + let row = window.row_off + local_row as u32; + let col = window.col_off + local_col as u32; + let row_no = row as usize * self.plan.width as usize + col as usize + 1; + let (x, y) = self.plan.grid.cell(col, row); + let mut coords = Vec::with_capacity(self.plan.coords.len()); + for source in &self.plan.coords { + coords.push(match source { + CoordSource::X => x.clone(), + CoordSource::Y => y.clone(), + CoordSource::Time(ms) => ArrayCoordLiteral::Int64(*ms), + }); + } + let mut attrs = Vec::with_capacity(self.plan.attrs.len()); + for source in &self.plan.attrs { + let AttrSource::Band { + band, + float, + nullable, + name, + } = source + else { + attrs.push(ArrayAttrLiteral::Null); + continue; + }; + let samples = self + .block + .bands + .get(*band) + .ok_or_else(|| missing_band(*band, row_no))?; + let is_nodata = self.plan.nodata.is_some_and(|n| n.matches(samples, index)); + let literal = match (is_nodata, *nullable) { + (true, true) => ArrayAttrLiteral::Null, + (true, false) => { + return Err(RowError::new( + row_no, + "23502", + format!( + "attr '{name}' is NOT NULL, and band {} holds nodata at \ + (pixel row {row}, col {col})", + band + 1 + ), + )); + } + (false, _) => sample_literal(samples, index, *float) + .ok_or_else(|| missing_band(*band, row_no))?, + }; + attrs.push(literal); + } + Ok((row_no, ArrayInsertRow { coords, attrs })) + } +} + +impl Iterator for BlockRows<'_> { + type Item = Result<(usize, ArrayInsertRow), RowError>; + + fn next(&mut self) -> Option { + while self.next < self.pixels { + let index = self.next; + self.next += 1; + if !self.is_nodata(index) { + return Some(self.row(index)); + } + } + None + } +} + +/// Sample `index` as an attr literal. Open admitted only lossless pairs, +/// so an integer sample widens to `f64` exactly when the attr is a float. +fn sample_literal(samples: &BandSamples, index: usize, float: bool) -> Option { + let int = |n: i64| { + if float { + ArrayAttrLiteral::Float64(n as f64) + } else { + ArrayAttrLiteral::Int64(n) + } + }; + match samples { + BandSamples::U8(v) => v.get(index).map(|s| int(i64::from(*s))), + BandSamples::U16(v) => v.get(index).map(|s| int(i64::from(*s))), + BandSamples::I16(v) => v.get(index).map(|s| int(i64::from(*s))), + BandSamples::U32(v) => v.get(index).map(|s| int(i64::from(*s))), + BandSamples::F32(v) => v + .get(index) + .map(|s| ArrayAttrLiteral::Float64(f64::from(*s))), + BandSamples::F64(v) => v.get(index).map(|s| ArrayAttrLiteral::Float64(*s)), + } +} + +fn missing_band(band: usize, row_no: usize) -> RowError { + RowError::new( + row_no, + "22P04", + format!("GeoTIFF band {} has no sample for this pixel", band + 1), + ) +} + +#[cfg(test)] +mod tests { + use super::super::grid::CellGrid; + use super::super::open::Nodata; + use super::*; + use nodedb_raster::BlockWindow; + + fn plan(nullable: bool) -> RasterPlan { + RasterPlan { + width: 4, + bands: 2, + coords: vec![CoordSource::Y, CoordSource::X], + attrs: vec![ + AttrSource::Band { + band: 0, + float: false, + nullable, + name: "b1".into(), + }, + AttrSource::Band { + band: 1, + float: true, + nullable: true, + name: "b2".into(), + }, + AttrSource::Null, + ], + loaded: vec![0, 1], + grid: CellGrid::Pixel { col0: 10, row0: 20 }, + nodata: Some(Nodata::Int(0)), + } + } + + fn block() -> RasterBlock { + RasterBlock { + index: 1, + window: BlockWindow { + col_off: 2, + row_off: 0, + width: 2, + height: 2, + }, + bands: vec![ + BandSamples::U16(vec![5, 0, 0, 7]), + BandSamples::U16(vec![6, 0, 9, 8]), + ], + } + } + + #[test] + fn rows_skip_all_nodata_pixels_and_null_partial_ones() { + let (plan, block) = (plan(true), block()); + let rows: Vec<_> = BlockRows::new(&plan, &block) + .expect("block fits") + .collect::>() + .expect("all rows load"); + let numbers: Vec = rows.iter().map(|(n, _)| *n).collect(); + // Pixel (0, 3) is nodata in both bands. Row-major ordinals: 2, 6, 7. + assert_eq!(numbers, vec![3, 7, 8]); + let (_, first) = &rows[0]; + assert_eq!( + first.coords, + vec![ArrayCoordLiteral::Int64(20), ArrayCoordLiteral::Int64(12)] + ); + assert_eq!( + first.attrs, + vec![ + ArrayAttrLiteral::Int64(5), + ArrayAttrLiteral::Float64(6.0), + ArrayAttrLiteral::Null + ] + ); + let (_, partial) = &rows[1]; + assert_eq!(partial.attrs[0], ArrayAttrLiteral::Null); + assert_eq!(partial.attrs[1], ArrayAttrLiteral::Float64(9.0)); + } + + #[test] + fn a_nodata_band_in_a_not_null_attr_names_the_pixel() { + let (plan, block) = (plan(false), block()); + let err = BlockRows::new(&plan, &block) + .expect("block fits") + .find_map(Result::err) + .expect("pixel (1, 2) is refused"); + assert_eq!(err.sqlstate, "23502"); + assert_eq!(err.row, 7); + assert!( + err.message.contains("(pixel row 1, col 2)"), + "{}", + err.message + ); + } + + #[test] + fn a_short_block_is_malformed() { + let plan = plan(true); + let mut block = block(); + block.bands[1] = BandSamples::U16(vec![1]); + let err = BlockRows::new(&plan, &block).err().expect("short band"); + assert_eq!(err.sqlstate, "22P04"); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/refusal.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/refusal.rs new file mode 100644 index 000000000..5a71e4ef4 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/refusal.rs @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The SQLSTATE of each GeoTIFF reader error. +//! +//! - A malformed or corrupt file is `22P04` (bad copy file format). +//! - A valid file that uses a feature the reader lacks is `0A000`. +//! - A block over the decode cap is `54000`. +//! - A read error outside the file's bytes is `58030`. + +use nodedb_raster::RasterError; + +use super::super::decode::RowError; + +/// `e`, met reading `path` under a `cap`-byte decode cap, as a refusal of +/// the whole input. +pub(super) fn raster_refusal(e: &RasterError, path: &str, cap: usize) -> RowError { + let sqlstate = sqlstate(e); + let message = if sqlstate == "54000" { + format!( + "'{path}': {e}; the decode cap is {cap} bytes; \ + retile the file with `gdal_translate -co TILED=YES`" + ) + } else { + format!("'{path}': {e}") + }; + RowError::new(0, sqlstate, message) +} + +fn sqlstate(e: &RasterError) -> &'static str { + match e { + RasterError::Malformed(_) + | RasterError::TagValueOutOfBounds { .. } + | RasterError::GeoKeys(_) + | RasterError::GeoTagLength { .. } + | RasterError::PixelScaleWithoutTiepoint + | RasterError::DegenerateTransform { .. } + | RasterError::MissingCrsKey { .. } + | RasterError::InvalidNodata { .. } + | RasterError::BlockOutOfRange { .. } + | RasterError::ShortBlock { .. } => "22P04", + RasterError::UnsupportedTiff(_) + | RasterError::UnsupportedCompression { .. } + | RasterError::UnsupportedSampleType { .. } + | RasterError::UnsupportedPhotometric { .. } + | RasterError::UnsupportedBandLayout { .. } + | RasterError::ReducedResolutionFirstImage + | RasterError::GroundControlPoints + | RasterError::NonAffineTransformation + | RasterError::UnsupportedModelType { .. } + | RasterError::UserDefinedCrs { .. } + | RasterError::UnsupportedCrs(_) => "0A000", + RasterError::BlockTooLarge { .. } | RasterError::DecodeLimitExceeded { .. } => "54000", + RasterError::Io(_) => "58030", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn each_class_maps_to_its_sqlstate() { + let cases = [ + (RasterError::PixelScaleWithoutTiepoint, "22P04"), + (RasterError::UnsupportedCompression { code: 7 }, "0A000"), + (RasterError::GroundControlPoints, "0A000"), + ( + RasterError::BlockTooLarge { + needed_bytes: 1 << 30, + limit_bytes: 1 << 26, + }, + "54000", + ), + (RasterError::Io(std::io::Error::other("disk gone")), "58030"), + ]; + for (error, code) in cases { + let refusal = raster_refusal(&error, "/data/scene.tif", 1 << 26); + assert_eq!(refusal.sqlstate, code, "{error}"); + assert_eq!(refusal.row, 0); + } + } + + #[test] + fn an_oversized_block_names_the_retile_command() { + let error = RasterError::DecodeLimitExceeded { limit_bytes: 4096 }; + let refusal = raster_refusal(&error, "/data/scene.tif", 4096); + assert!( + refusal.message.contains("gdal_translate -co TILED=YES"), + "{}", + refusal.message + ); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/task.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/task.rs new file mode 100644 index 000000000..00713adff --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/array_load/raster/task.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `COPY FROM '' WITH (FORMAT geotiff)`: the decode task and +//! the load that drains it. +//! +//! One blocking task owns the file handle and the decoder. It reads the +//! tags, waits for the open-time checks, then decodes one block at a time +//! into a channel of two. The async side turns each block into rows and +//! feeds the array loader, so the load holds at most four decoded blocks +//! and the loader's few chunks. Decoding runs on the Control Plane's +//! blocking pool, never on a Data Plane core. + +use std::io::BufReader; + +use nodedb_raster::{GeoTiffReader, RasterBlock, RasterError, RasterLimits}; +use nodedb_sql::ddl_ast::statement::CopyRasterOptions; +use tokio::sync::{mpsc, oneshot}; +use tokio::task::JoinHandle; + +use crate::control::server::shared::ddl::result::DdlError; + +use super::super::super::source_file::CopySourceFile; +use super::super::decode::RowError; +use super::super::dispatch::{ArrayLoadCtx, input_cap}; +use super::super::loader::{ArrayLoader, LoadInput}; +use super::open::{RasterHeader, RasterPlan}; +use super::pixels::BlockRows; +use super::refusal::raster_refusal; + +/// The most bytes one block decodes to, before the memory headroom cap. +const MAX_BLOCK_BYTES: usize = 64 * 1024 * 1024; +/// Decoded blocks queued between the decoder and the load. +const BLOCK_QUEUE: usize = 2; + +type HeaderResult = Result; +type BlockResult = Result; + +/// Load the GeoTIFF at `path` into `array`. Returns the number of rows +/// loaded: every pixel but the all-nodata ones. +pub(crate) async fn copy_raster( + ctx: &ArrayLoadCtx<'_>, + array: &str, + path: &str, + options: &CopyRasterOptions, +) -> Result { + let mut loader = ArrayLoader::open(ctx, array, LoadInput::Raster)?; + let georef = loader.view().georef.clone().ok_or_else(|| { + loader.refuse(RowError::new( + 0, + "55000", + format!( + "array '{array}' has no GEOREF, so no pixel has a cell; \ + add one with ALTER ARRAY {array} SET GEOREF (...)" + ), + )) + })?; + let cap = MAX_BLOCK_BYTES.min(input_cap(ctx.state)); + let file = CopySourceFile::open(path).await?.into_std().await; + let (header_tx, header_rx) = oneshot::channel(); + let (go_tx, go_rx) = oneshot::channel(); + let (block_tx, mut block_rx) = mpsc::channel(BLOCK_QUEUE); + let decoder = tokio::task::spawn_blocking(move || { + decode_file(file, cap, header_tx, go_rx, block_tx); + }); + + let header = match header_rx.await { + Ok(Ok(header)) => header, + Ok(Err(e)) => return Err(loader.refuse(raster_refusal(&e, path, cap))), + Err(_) => return Err(decoder_gone(decoder).await), + }; + let plan = + RasterPlan::new(loader.view(), &georef, &header, options).map_err(|e| loader.refuse(e))?; + if go_tx.send(()).is_err() { + return Err(decoder_gone(decoder).await); + } + while let Some(next) = block_rx.recv().await { + let block = next.map_err(|e| loader.refuse(raster_refusal(&e, path, cap)))?; + let rows = BlockRows::new(&plan, &block).map_err(|e| loader.refuse(e))?; + loader.feed_rows(ctx, rows).await?; + } + // The channel closes once the decoder returns. A panic closes it early, + // so the join result tells a whole file from a cut-short one. + decoder + .await + .map_err(|e| DdlError::internal(format!("COPY: GeoTIFF decoder failed: {e}")))?; + loader.finish(ctx).await +} + +/// The error of a decoder that ended before it answered. +async fn decoder_gone(decoder: JoinHandle<()>) -> DdlError { + match decoder.await { + Err(e) => DdlError::internal(format!("COPY: GeoTIFF decoder failed: {e}")), + Ok(()) => DdlError::internal("COPY: GeoTIFF decoder ended before the file was read"), + } +} + +/// The decode task. It ends early once the load drops its side: a refused +/// file, a refused row, or a cancelled statement. +fn decode_file( + file: std::fs::File, + cap: usize, + header_tx: oneshot::Sender, + go: oneshot::Receiver<()>, + blocks: mpsc::Sender, +) { + let opened = GeoTiffReader::open(BufReader::new(file), RasterLimits::new(cap)) + .and_then(|reader| header_of(&reader).map(|header| (reader, header))); + let mut reader = match opened { + Ok((reader, header)) => { + if header_tx.send(Ok(header)).is_err() { + return; + } + reader + } + Err(e) => { + // A dropped receiver means the load already ended; nothing waits. + let _ = header_tx.send(Err(e)); + return; + } + }; + if go.blocking_recv().is_err() { + return; + } + for index in 0..reader.block_count() { + let block = reader.read_block(index); + let failed = block.is_err(); + if blocks.blocking_send(block).is_err() || failed { + return; + } + } +} + +fn header_of( + reader: &GeoTiffReader, +) -> Result { + Ok(RasterHeader { + width: reader.width(), + height: reader.height(), + bands: reader.band_count(), + sample_type: reader.sample_type(), + transform: reader.transform().map(|gt| gt.0), + srid: reader.srid()?, + nodata: reader.nodata(), + }) +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/csv_import.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/csv_import.rs index 88dcc64c8..f10455e85 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/csv_import.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/csv_import.rs @@ -27,22 +27,20 @@ pub(super) struct CsvOptions { pub(super) has_header: bool, } -/// Import from a CSV file (header row → column names; each subsequent row → INSERT). +/// Import CSV `bytes`, read from `path` (header row → column names; each +/// subsequent row → INSERT). pub(super) async fn import_csv( ctx: &ImportCtx<'_>, collection: &str, path: &str, + bytes: &[u8], opts: CsvOptions, ) -> Result { let CsvOptions { delimiter, has_header, } = opts; - let bytes = tokio::fs::read(path) - .await - .map_err(|e| ddl_err("58030", format!("COPY: cannot read '{path}': {e}")))?; - - let content = std::str::from_utf8(&bytes).map_err(|e| { + let content = std::str::from_utf8(bytes).map_err(|e| { ddl_err( "22021", format!("COPY: file '{path}' is not valid UTF-8: {e}"), diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/entry.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/entry.rs index 18e940d55..4ec8966e6 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/entry.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/entry.rs @@ -1,22 +1,27 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Entry point: `copy_from_file`, path validation, and the engine's bulk-load -//! route, which `EngineRules::plan_bulk_load` answers. +//! Entry point: `copy_from_file`, the privilege and path checks, and the +//! engine's bulk-load route, which `EngineRules::plan_bulk_load` answers. +//! +//! A server-path `COPY` reads a file as the server's OS user, so only a +//! superuser may run it. The file is opened once, and every later check and +//! read uses that handle. //! //! The result type is [`DdlResult`] / [`DdlError`] throughout. Each error //! site builds a `DdlError` directly via [`ddl_err`] so the whole call chain //! speaks one error type. use nodedb_types::DatabaseId; -use std::path::Path; -use tokio::io::AsyncReadExt; use nodedb_sql::SqlError; -use nodedb_sql::ddl_ast::statement::{CopyFormat, CopyStdinFormat}; +use nodedb_sql::ddl_ast::statement::{CopyFormat, CopyRasterOptions, CopyStdinFormat}; use crate::control::planner::context::{BulkLoadRoute, QueryContext}; use crate::control::security::identity::AuthenticatedIdentity; use crate::control::server::shared::ddl::neutral::collection::dml::authorize_write_target; +use crate::control::server::shared::ddl::neutral::collection::server_path::{ + require_superuser, validate_path, +}; use crate::control::server::shared::ddl::result::{DdlError, DdlResult}; use crate::control::server::shared::session::DmlTxnCtx; use crate::control::state::SharedState; @@ -24,23 +29,31 @@ use crate::control::state::SharedState; use crate::control::trigger::statement_txn::{fires_joined_body, in_block, with_statement_txn}; use crate::control::trigger::{DmlEvent, TriggerScope}; +use super::array_load::raster::copy_raster; use super::array_load::{ArrayLoadCtx, ArrayLoader, LoadInput, TextFormat}; use super::csv_import::{CsvOptions, import_csv}; use super::import_ctx::ImportCtx; use super::json_import::{import_json_array, import_ndjson}; - -/// Maximum file size accepted for COPY FROM (16 GiB). -pub(super) const MAX_FILE_BYTES: u64 = 16 * 1024 * 1024 * 1024; +use super::source_file::CopySourceFile; /// How many bytes one read of an array load's file takes. const FILE_READ_BYTES: usize = 1024 * 1024; -/// COPY FROM format and delimiter options. +/// COPY FROM format, text, and GeoTIFF options. #[derive(Clone, Copy, Debug)] pub struct CopyFromOptions<'a> { pub format: Option<&'a CopyFormat>, pub delimiter: Option, pub header: bool, + /// `BANDS`, `NODATA` and `TIME`. Empty unless `format` is `GeoTiff`. + pub raster: &'a CopyRasterOptions, +} + +/// The row format of a collection load. +enum RowFormat { + Ndjson, + JsonArray, + Csv(CsvOptions), } /// Execute `COPY FROM '' [WITH (...)]`. @@ -57,25 +70,12 @@ pub async fn copy_from_file( format, delimiter, header, + raster: _, } = options; + require_superuser(identity, "COPY ... FROM ''")?; authorize_write_target(state, identity, database_id, collection)?; - validate_path(path)?; - - // Check file size before reading. - let metadata = tokio::fs::metadata(path) - .await - .map_err(|e| ddl_err("58030", format!("COPY: cannot stat file '{path}': {e}")))?; - if metadata.len() > MAX_FILE_BYTES { - return Err(ddl_err( - "54000", - format!( - "COPY: file '{path}' is {} bytes, exceeds limit of {} bytes", - metadata.len(), - MAX_FILE_BYTES - ), - )); - } + validate_path(path, "COPY")?; // Determine format (caller has already auto-detected from extension; this is a safety net). let resolved_format = format.ok_or_else(|| { @@ -83,7 +83,7 @@ pub async fn copy_from_file( "42601", format!( "COPY: cannot infer format for '{path}'; \ - add WITH (FORMAT ndjson|json|csv)" + add WITH (FORMAT ndjson|json|csv|geotiff)" ), ) })?; @@ -105,6 +105,22 @@ pub async fn copy_from_file( return copy_array_from_file(&ctx, collection, path, options).await; } + let row_format = match resolved_format { + CopyFormat::Ndjson => RowFormat::Ndjson, + CopyFormat::JsonArray => RowFormat::JsonArray, + CopyFormat::Csv => RowFormat::Csv(CsvOptions { + delimiter: delimiter.unwrap_or(','), + has_header: header, + }), + CopyFormat::GeoTiff => { + return Err(ddl_err( + "0A000", + format!("COPY: FORMAT geotiff loads into an array; '{collection}' is not an array"), + )); + } + }; + let bytes = CopySourceFile::open(path).await?.read_all().await?; + // Each row fires its collection's BEFORE, INSTEAD OF and SYNC AFTER // bodies, which join the statement's transaction. Outside a transaction // block a COPY into such a collection runs in an implicit one: its rows @@ -132,21 +148,12 @@ pub async fn copy_from_file( database_id, txn_ctx, }; - match resolved_format { - CopyFormat::Ndjson => import_ndjson(&import_ctx, collection, path).await, - CopyFormat::JsonArray => import_json_array(&import_ctx, collection, path).await, - CopyFormat::Csv => { - import_csv( - &import_ctx, - collection, - path, - CsvOptions { - delimiter: delimiter.unwrap_or(','), - has_header: header, - }, - ) - .await + match row_format { + RowFormat::Ndjson => import_ndjson(&import_ctx, collection, path, &bytes).await, + RowFormat::JsonArray => { + import_json_array(&import_ctx, collection, path, &bytes).await } + RowFormat::Csv(csv) => import_csv(&import_ctx, collection, path, &bytes, csv).await, } }, ) @@ -164,33 +171,6 @@ pub async fn copy_from_file( }]) } -/// Reject paths with `..` segments and non-absolute paths. -fn validate_path(path: &str) -> Result<(), DdlError> { - if !path.starts_with('/') { - return Err(ddl_err( - "42601", - format!( - "COPY: path '{path}' is not absolute; \ - only absolute server-side paths are accepted" - ), - )); - } - let p = Path::new(path); - for component in p.components() { - use std::path::Component; - if matches!(component, Component::ParentDir) { - return Err(ddl_err( - "42501", - format!( - "COPY: path '{path}' contains '..'; \ - directory traversal is not permitted" - ), - )); - } - } - Ok(()) -} - /// The DDL error of a refused bulk-load route. An engine's refusal keeps its /// exact text under `0A000`. pub(super) fn bulk_load_refusal(e: SqlError, tenant_id: nodedb_types::TenantId) -> DdlError { @@ -202,7 +182,7 @@ pub(super) fn bulk_load_refusal(e: SqlError, tenant_id: nodedb_types::TenantId) /// `COPY FROM ''`: the file's CSV rows stream through the /// array bulk-load core in reads of [`FILE_READ_BYTES`], as -/// `COPY FROM STDIN` does. +/// `COPY FROM STDIN` does. A GeoTIFF streams its decoded blocks. async fn copy_array_from_file( ctx: &ArrayLoadCtx<'_>, array: &str, @@ -213,25 +193,34 @@ async fn copy_array_from_file( format, delimiter, header, + raster, } = options; - if format != Some(&CopyFormat::Csv) { - return Err(ddl_err( - "0A000", - format!("COPY: array '{array}' loads CSV rows; use WITH (FORMAT csv)"), - )); + match format { + Some(CopyFormat::Csv) => {} + Some(CopyFormat::GeoTiff) => { + let rows = copy_raster(ctx, array, path, raster).await?; + return Ok(vec![DdlResult::Status { + command: format!("COPY {rows}"), + rows_affected: None, + }]); + } + Some(CopyFormat::Ndjson | CopyFormat::JsonArray) | None => { + return Err(ddl_err( + "0A000", + format!("COPY: array '{array}' loads CSV rows; use WITH (FORMAT csv)"), + )); + } } let format = TextFormat { format: CopyStdinFormat::Csv, delimiter, header, }; + let mut source = CopySourceFile::open(path).await?; let mut loader = ArrayLoader::open(ctx, array, LoadInput::Text(format))?; - let read_error = - |e: std::io::Error| ddl_err("58030", format!("COPY: cannot read '{path}': {e}")); - let mut file = tokio::fs::File::open(path).await.map_err(read_error)?; let mut buf = vec![0u8; FILE_READ_BYTES]; loop { - let n = file.read(&mut buf).await.map_err(read_error)?; + let n = source.read_some(&mut buf).await?; if n == 0 { break; } diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/json_import.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/json_import.rs index bf1d83965..c9224c3b7 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/json_import.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/json_import.rs @@ -20,7 +20,8 @@ fn ddl_err(sqlstate: &str, message: impl Into) -> DdlError { DdlError::new(sqlstate, message) } -/// Import from an NDJSON file (one JSON object per non-empty line). +/// Import NDJSON `bytes`, read from `path` (one JSON object per non-empty +/// line). /// /// Parse-then-insert: all rows are parsed before any INSERT is issued so that /// a JSON parse error on any line prevents rows from being partially written. @@ -28,12 +29,9 @@ pub(super) async fn import_ndjson( ctx: &ImportCtx<'_>, collection: &str, path: &str, + bytes: &[u8], ) -> Result { - let bytes = tokio::fs::read(path) - .await - .map_err(|e| ddl_err("58030", format!("COPY: cannot read '{path}': {e}")))?; - - let content = std::str::from_utf8(&bytes).map_err(|e| { + let content = std::str::from_utf8(bytes).map_err(|e| { ddl_err( "22021", format!("COPY: file '{path}' is not valid UTF-8: {e}"), @@ -75,7 +73,7 @@ pub(super) async fn import_ndjson( Ok(parsed.len()) } -/// Import from a JSON array file (`[{...}, {...}, ...]`). +/// Import JSON array `bytes`, read from `path` (`[{...}, {...}, ...]`). /// /// Parse-then-insert: all rows are deserialized before any INSERT so that /// a malformed array prevents partial writes. @@ -83,12 +81,9 @@ pub(super) async fn import_json_array( ctx: &ImportCtx<'_>, collection: &str, path: &str, + bytes: &[u8], ) -> Result { - let bytes = tokio::fs::read(path) - .await - .map_err(|e| ddl_err("58030", format!("COPY: cannot read '{path}': {e}")))?; - - let array: Vec = sonic_rs::from_slice(&bytes).map_err(|e| { + let array: Vec = sonic_rs::from_slice(bytes).map_err(|e| { ddl_err( "22P02", format!("COPY: file '{path}' is not a valid JSON array: {e}"), diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/mod.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/mod.rs index 05cc79193..a4b91b430 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/mod.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/mod.rs @@ -4,8 +4,9 @@ //! //! `COPY FROM ''` reads NDJSON, JSON array, or CSV. The //! format is detected from the file extension when the WITH clause names -//! none. `COPY FROM STDIN` and `COPY FROM '.csv'` load -//! an array through `array_load`. +//! none. A server path needs superuser, and the path must open as a regular +//! file. `COPY FROM STDIN`, `COPY FROM '.csv'` and +//! `COPY FROM '.tif'` load an array through `array_load`. //! //! Each engine's `EngineRules::plan_bulk_load` decides how it loads: //! - Document (schemaless/strict), Columnar, KV, vector primary: one INSERT @@ -28,6 +29,7 @@ mod csv_import; mod entry; mod import_ctx; mod json_import; +mod source_file; pub(crate) use array_load::{ArrayLoadCtx, ArrayLoader, LoadInput, TextFormat}; pub use entry::{CopyFromOptions, copy_from_file}; diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/source_file.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/source_file.rs new file mode 100644 index 000000000..b7d250663 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_from/source_file.rs @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The server file a `COPY ... FROM ''` reads, opened once. +//! +//! The open is non-blocking, so a FIFO or a device never stalls it. Every +//! check then runs on the open handle: the file type, the size cap, and +//! each read. No second path lookup can resolve to another file. + +use tokio::io::AsyncReadExt; + +use crate::control::server::shared::ddl::result::DdlError; + +/// Maximum file size accepted for COPY FROM (16 GiB). +const MAX_FILE_BYTES: u64 = 16 * 1024 * 1024 * 1024; + +/// An open regular file under the size cap. +pub(crate) struct CopySourceFile { + file: tokio::fs::File, + path: String, + /// The size `fstat` reported at open. + len: u64, + /// Bytes `read_some` returned so far. + consumed: u64, +} + +impl CopySourceFile { + /// Open `path` and check it through the handle. A missing or unreadable + /// path and a non-regular file are `58030`. A file over the cap is + /// `54000`. + pub(crate) async fn open(path: &str) -> Result { + let file = tokio::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NONBLOCK) + .open(path) + .await + .map_err(|e| DdlError::new("58030", format!("COPY: cannot open '{path}': {e}")))?; + let metadata = file + .metadata() + .await + .map_err(|e| DdlError::new("58030", format!("COPY: cannot stat '{path}': {e}")))?; + if !metadata.is_file() { + return Err(DdlError::new( + "58030", + format!( + "COPY: '{path}' is not a regular file; \ + COPY reads regular files only, never a FIFO, device, or directory" + ), + )); + } + let len = metadata.len(); + if len > MAX_FILE_BYTES { + return Err(too_large(path, len)); + } + // A read of a regular file never blocks, so `O_NONBLOCK` changes + // nothing past the open. + Ok(Self { + file, + path: path.to_owned(), + len, + consumed: 0, + }) + } + + /// The open handle as a blocking file, for a decoder that runs on a + /// blocking thread. The handle is the one every open-time check ran on. + pub(crate) async fn into_std(self) -> std::fs::File { + self.file.into_std().await + } + + /// Read the next bytes into `buf`, `0` at the end of the file. A file + /// that grew past the cap after the open is `54000`. + pub(crate) async fn read_some(&mut self, buf: &mut [u8]) -> Result { + let read = self.file.read(buf).await; + let n = read.map_err(|e| self.read_error(e))?; + self.consumed = self.consumed.saturating_add(n as u64); + if self.consumed > MAX_FILE_BYTES { + return Err(too_large(&self.path, self.consumed)); + } + Ok(n) + } + + /// Read the whole file through the handle. A file that grew past the + /// cap after the open is `54000`. + pub(crate) async fn read_all(&mut self) -> Result, DdlError> { + let hint = usize::try_from(self.len).unwrap_or(0); + let mut bytes = Vec::with_capacity(hint); + let read = (&mut self.file) + .take(MAX_FILE_BYTES + 1) + .read_to_end(&mut bytes) + .await; + read.map_err(|e| self.read_error(e))?; + let total = bytes.len() as u64; + if total > MAX_FILE_BYTES { + return Err(too_large(&self.path, total)); + } + Ok(bytes) + } + + /// The `58030` error of a failed read. + fn read_error(&self, e: std::io::Error) -> DdlError { + DdlError::new("58030", format!("COPY: cannot read '{}': {e}", self.path)) + } +} + +fn too_large(path: &str, len: u64) -> DdlError { + DdlError::new( + "54000", + format!("COPY: file '{path}' is {len} bytes, exceeds limit of {MAX_FILE_BYTES} bytes"), + ) +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_to/entry.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_to/entry.rs index 449a86a67..e791e39b7 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_to/entry.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_to/entry.rs @@ -5,7 +5,7 @@ //! The result type is [`DdlResult`] / [`DdlError`] throughout. use nodedb_types::DatabaseId; -use std::path::Path; +use tokio::io::AsyncWriteExt; use sonic_rs; @@ -15,6 +15,9 @@ use crate::control::security::identity::AuthenticatedIdentity; use crate::control::security::redaction::RedactionStore; use crate::control::security::request_scope::RequestAuthScope; use crate::control::server::response_shape::redaction::{QueryRedaction, redact_decoded_value}; +use crate::control::server::shared::ddl::neutral::collection::server_path::{ + require_superuser, validate_path, +}; use crate::control::server::shared::ddl::result::{DdlError, DdlResult}; use crate::control::state::SharedState; use crate::types::TraceId; @@ -49,7 +52,8 @@ pub async fn copy_to_file( header, } = options; - validate_path(path)?; + require_superuser(identity, "COPY ... TO ''")?; + validate_path(path, "COPY TO")?; // Resolve format (caller has already auto-detected from extension). let resolved_format = format.ok_or_else(|| { @@ -76,9 +80,10 @@ pub async fn copy_to_file( // Serialize to the requested format. let bytes = serialize_rows(&rows, resolved_format, delimiter.unwrap_or(','), header)?; - // Atomic write: temp file → rename. + // Atomic write: temp file → rename. The temp file must not exist yet, + // so a planted symlink or FIFO at that name is never followed. let tmp_path = format!("{path}.tmp"); - tokio::fs::write(&tmp_path, &bytes).await.map_err(|e| { + write_new_file(&tmp_path, &bytes).await.map_err(|e| { ddl_err( "58030", format!("COPY TO: cannot write to '{tmp_path}': {e}"), @@ -237,31 +242,25 @@ fn extract_json_rows( Ok(()) } -/// Reject paths with `..` segments and non-absolute paths. -fn validate_path(path: &str) -> Result<(), DdlError> { - if !path.starts_with('/') { - return Err(ddl_err( - "42601", - format!( - "COPY TO: path '{path}' is not absolute; \ - only absolute server-side paths are accepted" - ), - )); +/// Create `path`, which must not exist, and write `bytes` to it. A failed +/// write removes the file it created. +async fn write_new_file(path: &str, bytes: &[u8]) -> std::io::Result<()> { + let mut file = tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .await?; + let written = match file.write_all(bytes).await { + Ok(()) => file.flush().await, + Err(e) => Err(e), + }; + if written.is_err() { + drop(file); + // The write error is the one reported. A failed removal leaves a + // temp file the next COPY TO to this path refuses with its name. + let _ = tokio::fs::remove_file(path).await; } - let p = Path::new(path); - for component in p.components() { - use std::path::Component; - if matches!(component, Component::ParentDir) { - return Err(ddl_err( - "42501", - format!( - "COPY TO: path '{path}' contains '..'; \ - directory traversal is not permitted" - ), - )); - } - } - Ok(()) + written } #[cfg(test)] diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_to/format.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_to/format.rs index 4eea69e17..8cb4b16dc 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/copy_to/format.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/copy_to/format.rs @@ -26,6 +26,11 @@ pub(super) fn serialize_rows( CopyFormat::Ndjson => serialize_ndjson(rows), CopyFormat::JsonArray => serialize_json_array(rows), CopyFormat::Csv => serialize_csv(rows, delimiter, header), + // The parser refuses a GeoTIFF export, so no plan names one. + CopyFormat::GeoTiff => Err(ddl_err( + "0A000", + "COPY TO does not write GeoTIFF; export with FORMAT ndjson, json, or csv", + )), } } diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/describe.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/describe.rs index b39c4f5f4..be45ea918 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/describe.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/describe.rs @@ -54,6 +54,11 @@ pub fn describe_collection( let coll = match catalog.get_collection(database_id, tenant_id.as_u64(), name) { Ok(Some(c)) if c.is_active => c, _ => { + if let Some(described) = + super::describe_array::describe_array(state, tenant_id, database_id, name) + { + return described; + } return Err(DdlError::new( "42P01", format!("collection '{name}' not found"), diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/describe_array.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/describe_array.rs new file mode 100644 index 000000000..aceb431cd --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/describe_array.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! DESCRIBE for an array (`CREATE ARRAY`). +//! +//! Arrays live in the array catalog, not in `StoredCollection`. The result +//! uses the DESCRIBE COLLECTION columns: one row per dim, one row per attr, +//! then `__storage`, `__tile_extents`, and `__georef` when the array has one. + +use nodedb_array::schema::{ArraySchema, DimType}; +use nodedb_array::types::domain::DomainBound; +use nodedb_types::{DatabaseId, TenantId}; +use serde_json::{Map, Value as JsonValue}; + +use crate::control::array_catalog::georef::{georef_ast, georef_sql}; +use crate::control::server::response_shape::types::{DdlColType, ShapedRows}; +use crate::control::state::SharedState; + +use super::super::super::result::{DdlError, DdlResult}; + +/// DESCRIBE rows for array `name`, or `None` when no array has that name. +pub(super) fn describe_array( + state: &SharedState, + tenant_id: TenantId, + database_id: DatabaseId, + name: &str, +) -> Option, DdlError>> { + let entry = state + .array_catalog + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .lookup_by_name_in_database(tenant_id, database_id, name)?; + let schema: ArraySchema = match zerompk::from_msgpack(&entry.schema_msgpack) { + Ok(schema) => schema, + Err(e) => { + return Some(Err(DdlError::new( + "XX000", + format!("array '{name}': stored schema decode: {e}"), + ))); + } + }; + Some(Ok(vec![DdlResult::Rows(ShapedRows::from_json_rows( + vec![ + "field".to_string(), + "type".to_string(), + "nullable".to_string(), + "primary_key".to_string(), + ], + vec![ + DdlColType::Text, + DdlColType::Text, + DdlColType::Text, + DdlColType::Bool, + ], + rows(&schema), + ))])) +} + +/// The DESCRIBE rows of `schema`. +fn rows(schema: &ArraySchema) -> Vec> { + let mut rows = Vec::with_capacity(schema.dims.len() + schema.attrs.len() + 3); + for dim in &schema.dims { + let ty = format!( + "{} DIM [{}..{}]", + dim_type_name(dim.dtype), + bound_text(&dim.domain.lo), + bound_text(&dim.domain.hi) + ); + rows.push(row(&dim.name, &ty, "false")); + } + for attr in &schema.attrs { + let nullable = if attr.nullable { "true" } else { "false" }; + rows.push(row(&attr.name, attr.dtype.sql_name(), nullable)); + } + rows.push(row("__storage", "array", "false")); + let extents: Vec = schema.tile_extents.iter().map(u64::to_string).collect(); + rows.push(row("__tile_extents", &extents.join(", "), "false")); + if let Some(georef) = georef_ast(schema) { + rows.push(row("__georef", &georef_sql(&georef), "false")); + } + rows +} + +fn row(field: &str, ty: &str, nullable: &str) -> Map { + let mut row = Map::new(); + row.insert("field".to_string(), JsonValue::String(field.to_string())); + row.insert("type".to_string(), JsonValue::String(ty.to_string())); + row.insert( + "nullable".to_string(), + JsonValue::String(nullable.to_string()), + ); + row.insert("primary_key".to_string(), JsonValue::Bool(false)); + row +} + +fn dim_type_name(dtype: DimType) -> &'static str { + match dtype { + DimType::Int64 => "INT64", + DimType::Float64 => "FLOAT64", + DimType::TimestampMs => "TIMESTAMP_MS", + DimType::String => "STRING", + } +} + +fn bound_text(bound: &DomainBound) -> String { + match bound { + DomainBound::Int64(v) | DomainBound::TimestampMs(v) => v.to_string(), + DomainBound::Float64(v) => format!("{v:?}"), + DomainBound::String(v) => format!("'{}'", v.replace('\'', "''")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_array::schema::{ + ArrayGeoref, ArraySchemaBuilder, AttrSpec, AttrType, DimSpec, GeoTransform, + }; + use nodedb_array::types::domain::Domain; + + fn schema(georef: Option) -> ArraySchema { + let dim = |name: &str| { + DimSpec::new( + name, + DimType::Int64, + Domain::new(DomainBound::Int64(0), DomainBound::Int64(1023)), + ) + }; + let mut builder = ArraySchemaBuilder::new("scene") + .dim(dim("row")) + .dim(dim("col")) + .attr(AttrSpec::new("b1", AttrType::UInt16, false)) + .tile_extents(vec![256, 256]); + if let Some(georef) = georef { + builder = builder.georef(georef); + } + builder.build().unwrap() + } + + fn field(rows: &[Map], name: &str) -> Option { + rows.iter() + .find(|r| r.get("field") == Some(&JsonValue::String(name.into()))) + .and_then(|r| r.get("type")) + .and_then(|t| t.as_str().map(str::to_string)) + } + + #[test] + fn rows_list_dims_attrs_and_storage() { + let rows = rows(&schema(None)); + assert_eq!(field(&rows, "row").as_deref(), Some("INT64 DIM [0..1023]")); + assert_eq!(field(&rows, "b1").as_deref(), Some("UINT16")); + assert_eq!(field(&rows, "__storage").as_deref(), Some("array")); + assert_eq!(field(&rows, "__tile_extents").as_deref(), Some("256, 256")); + assert_eq!(field(&rows, "__georef"), None); + } + + #[test] + fn rows_show_the_georef() { + let rows = rows(&schema(Some(ArrayGeoref { + srid: nodedb_types::geometry::Srid::Utm47N, + x_dim: 1, + y_dim: 0, + transform: Some(GeoTransform([600_000.0, 10.0, 0.0, 700_000.0, 0.0, -10.0])), + }))); + assert_eq!( + field(&rows, "__georef").as_deref(), + Some("SRID 32647, X col, Y row, TRANSFORM (600000.0, 10.0, 0.0, 700000.0, 0.0, -10.0)") + ); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/mod.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/mod.rs index 1bd52e958..4dcce3f99 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/mod.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/mod.rs @@ -10,6 +10,7 @@ pub mod copy_from; pub mod copy_to; pub mod create; pub mod describe; +mod describe_array; pub mod dml; pub mod drop; pub mod enforcement; @@ -18,6 +19,7 @@ pub mod index; pub(super) mod index_fanout; pub mod purge; pub mod register; +pub(crate) mod server_path; pub mod show_indexes; pub mod undrop; pub mod vector_metadata; diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/server_path.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/server_path.rs new file mode 100644 index 000000000..90ccb3b20 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/server_path.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Checks a server-path `COPY` runs before it touches the filesystem. +//! +//! A server path names a file the server's OS user reads or writes. A +//! NodeDB grant carries no filesystem right, so only a superuser can name +//! one. + +use std::path::{Component, Path}; + +use crate::control::security::identity::AuthenticatedIdentity; +use crate::control::server::shared::ddl::result::DdlError; + +/// Refuse a non-superuser with `42501`. `statement` names the form, such +/// as `COPY ... FROM ''`. +pub(crate) fn require_superuser( + identity: &AuthenticatedIdentity, + statement: &str, +) -> Result<(), DdlError> { + if identity.is_superuser { + return Ok(()); + } + Err(DdlError::new( + "42501", + format!( + "permission denied: {statement} accesses a file as the server's OS user \ + and requires superuser" + ), + )) +} + +/// Refuse a relative path (`42601`) and a path with a `..` component +/// (`42501`). `statement` prefixes the message, such as `COPY TO`. +pub(crate) fn validate_path(path: &str, statement: &str) -> Result<(), DdlError> { + if !path.starts_with('/') { + return Err(DdlError::new( + "42601", + format!( + "{statement}: path '{path}' is not absolute; \ + only absolute server-side paths are accepted" + ), + )); + } + if Path::new(path) + .components() + .any(|component| matches!(component, Component::ParentDir)) + { + return Err(DdlError::new( + "42501", + format!( + "{statement}: path '{path}' contains '..'; \ + directory traversal is not permitted" + ), + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn relative_and_traversal_paths_are_refused() { + assert_eq!( + validate_path("data/x.csv", "COPY") + .expect_err("relative") + .sqlstate, + "42601" + ); + assert_eq!( + validate_path("/tmp/../etc/passwd", "COPY") + .expect_err("traversal") + .sqlstate, + "42501" + ); + assert!(validate_path("/srv/data/x.csv", "COPY").is_ok()); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/column_default.rs b/nodedb/src/control/server/shared/ddl/neutral/column_default.rs index ebaba05fe..a8ca85ebf 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/column_default.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/column_default.rs @@ -167,7 +167,8 @@ fn clause_error(clause: &str, owner: &str, error: &SqlError) -> DdlError { SqlError::SequencePerRowUnsupported { .. } | SqlError::SearchFunctionOutsideSearch { .. } | SqlError::UnsupportedConstraint { .. } - | SqlError::ConflictingEngineClause { .. } => sqlstate::FEATURE_NOT_SUPPORTED, + | SqlError::ConflictingEngineClause { .. } + | SqlError::FeatureNotSupported { .. } => sqlstate::FEATURE_NOT_SUPPORTED, SqlError::RetryableSchemaChanged { .. } => sqlstate::SERIALIZATION_FAILURE, SqlError::RecursionDepthExceeded { .. } => sqlstate::PROGRAM_LIMIT_EXCEEDED, SqlError::Parse { .. } diff --git a/nodedb/src/control/server/shared/ddl/neutral/router/typed_misc.rs b/nodedb/src/control/server/shared/ddl/neutral/router/typed_misc.rs index d61a25cfa..1d77f517d 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/router/typed_misc.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/router/typed_misc.rs @@ -31,6 +31,7 @@ pub(super) async fn try_typed( format, delimiter, header, + raster, }) => Some( collection::copy_from_file( state, @@ -41,6 +42,7 @@ pub(super) async fn try_typed( format: format.as_ref(), delimiter: *delimiter, header: *header, + raster, }, database_id, txn_ctx, diff --git a/nodedb/src/control/server/shared/ddl/result.rs b/nodedb/src/control/server/shared/ddl/result.rs index 93e72115d..eb5b64413 100644 --- a/nodedb/src/control/server/shared/ddl/result.rs +++ b/nodedb/src/control/server/shared/ddl/result.rs @@ -262,10 +262,12 @@ pub fn code_for_sqlstate(sqlstate_str: &str) -> ErrorCode { // non-retriable client error. sqlstate::GROUPING_ERROR => ErrorCode::BAD_REQUEST, sqlstate::DATA_EXCEPTION => ErrorCode::DATA_EXCEPTION, - // A bad parameter value, text representation, datetime format or - // datetime range is a data exception: class `22`, the class the code - // renders back. - "22023" | "22P02" | "22007" | "22008" => ErrorCode::DATA_EXCEPTION, + // A bad parameter value, text representation, datetime format, + // datetime range or COPY file format is a data exception: class + // `22`, the class the code renders back. + "22023" | "22P02" | "22007" | "22008" | sqlstate::BAD_COPY_FILE_FORMAT => { + ErrorCode::DATA_EXCEPTION + } sqlstate::NUMERIC_VALUE_OUT_OF_RANGE => ErrorCode::OVERFLOW, sqlstate::DIVISION_BY_ZERO => ErrorCode::DIVISION_BY_ZERO, sqlstate::INVALID_LIMIT_VALUE => ErrorCode::INVALID_LIMIT_VALUE, diff --git a/nodedb/src/control/server/shared/sql/staging_predicates.rs b/nodedb/src/control/server/shared/sql/staging_predicates.rs index 596934b26..57f68dbda 100644 --- a/nodedb/src/control/server/shared/sql/staging_predicates.rs +++ b/nodedb/src/control/server/shared/sql/staging_predicates.rs @@ -719,6 +719,7 @@ mod tests { shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, + zone_msgpack: Vec::new(), }); assert!(!is_stageable_write(&slice)); } diff --git a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs index 8a659ae60..c3bfadaf3 100644 --- a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs +++ b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs @@ -1978,6 +1978,7 @@ mod tests { array_id: "a".into(), audit_retain_ms: None, minimum_audit_retain_ms: None, + schema_msgpack: None, }), PhysicalPlan::Meta(MetaOp::ApplyContinuousAggRetention), PhysicalPlan::Meta(MetaOp::QueryAggregateWatermark { @@ -2086,6 +2087,7 @@ mod tests { shard_bucket: None, system_time: SystemTimeScope::Current, valid_at_ms: None, + zone_msgpack: Vec::new(), }), PhysicalPlan::Array(ArrayOp::Aggregate { array_id: array_id.clone(), @@ -2149,6 +2151,7 @@ mod tests { prefix_bits: 0, system_time: SystemTimeScope::Current, valid_at_ms: None, + zone_msgpack: Vec::new(), }), PhysicalPlan::ClusterArray(ClusterArrayOp::Agg { array_id: array_id.clone(), diff --git a/nodedb/src/data/executor/array_checkpoint.rs b/nodedb/src/data/executor/array_checkpoint.rs index b77527ec2..372d66c1c 100644 --- a/nodedb/src/data/executor/array_checkpoint.rs +++ b/nodedb/src/data/executor/array_checkpoint.rs @@ -340,6 +340,7 @@ mod tests { shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, + zone_msgpack: Vec::new(), }); assert_eq!(r.status, Status::Ok, "array slice: {r:?}"); decode_cells(r.payload.as_bytes()) diff --git a/nodedb/src/data/executor/core_loop/write_index/validate.rs b/nodedb/src/data/executor/core_loop/write_index/validate.rs index be11d6da6..e1473649b 100644 --- a/nodedb/src/data/executor/core_loop/write_index/validate.rs +++ b/nodedb/src/data/executor/core_loop/write_index/validate.rs @@ -258,6 +258,7 @@ mod tests { shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, + zone_msgpack: Vec::new(), }, )) }) diff --git a/nodedb/src/data/executor/dispatch/array/aggregate.rs b/nodedb/src/data/executor/dispatch/array/aggregate.rs index b0efe9177..197dc6117 100644 --- a/nodedb/src/data/executor/dispatch/array/aggregate.rs +++ b/nodedb/src/data/executor/dispatch/array/aggregate.rs @@ -227,12 +227,14 @@ impl CoreLoop { &mut resolved_tiles, ) .map_err(|e| format!("array aggregate overlay merge: {e}"))?; + // One prepared region per scan: a zone's transformer is built once. + let prepared = region.map(ArrayRegion::prepare); let mut tiles = Vec::with_capacity(resolved_tiles.len()); for (hp, tile) in resolved_tiles { if !shard_bucket_admits(shard_bucket, hp) { continue; } - let tile = match region { + let tile = match &prepared { Some(region) => filter_region_owned(schema, tile, region) .map_err(|e| format!("array aggregate region filter: {e}"))?, None => tile, @@ -836,6 +838,7 @@ mod tests { x_dim: 0, y_dim: 1, polygons: vec![ZonePolygon { exterior, holes }], + frame: None, }), ) } diff --git a/nodedb/src/data/executor/dispatch/array/entry.rs b/nodedb/src/data/executor/dispatch/array/entry.rs index 19d0e22ac..f1f33ced3 100644 --- a/nodedb/src/data/executor/dispatch/array/entry.rs +++ b/nodedb/src/data/executor/dispatch/array/entry.rs @@ -92,6 +92,7 @@ impl CoreLoop { shard_bucket, system_time, valid_at_ms, + zone_msgpack, } => self.dispatch_array_slice( task, SliceParams { @@ -103,6 +104,7 @@ impl CoreLoop { shard_bucket: *shard_bucket, system_time: *system_time, valid_at_ms: *valid_at_ms, + zone_msgpack, }, ), ArrayOp::SurrogateBitmapScan { diff --git a/nodedb/src/data/executor/dispatch/array/read.rs b/nodedb/src/data/executor/dispatch/array/read.rs index a45d60574..2dbd0eb46 100644 --- a/nodedb/src/data/executor/dispatch/array/read.rs +++ b/nodedb/src/data/executor/dispatch/array/read.rs @@ -11,8 +11,8 @@ //! row is the named map `slice_row::named_cell_row` builds. use nodedb_array::query::project::{Projection, project_sparse}; -use nodedb_array::query::region::ArrayRegion; -use nodedb_array::query::slice::{Slice, slice_sparse_owned, tile_overlaps_slice}; +use nodedb_array::query::region::{ArrayRegion, PolygonZone}; +use nodedb_array::query::slice::{Slice, filter_sparse, slice_sparse_owned}; use nodedb_array::types::ArrayId; use nodedb_types::array_route::shard_bucket_admits; use nodedb_types::{ShardBucket, SurrogateBitmap, SystemTimeScope, Value}; @@ -37,6 +37,8 @@ pub(in crate::data::executor) struct SliceParams<'a> { pub system_time: SystemTimeScope, /// Bitemporal valid-time point. `None` = no valid-time filter. pub valid_at_ms: Option, + /// zerompk-encoded [`PolygonZone`]. Empty = no zone. + pub zone_msgpack: &'a [u8], } use crate::bridge::envelope::{ErrorCode, Response}; @@ -63,6 +65,7 @@ impl CoreLoop { shard_bucket, system_time, valid_at_ms, + zone_msgpack, } = p; if let Err(resp) = self.ensure_array_open(task, array_id) { @@ -79,6 +82,25 @@ impl CoreLoop { ); } }; + let zone: Option = if zone_msgpack.is_empty() { + None + } else { + match zerompk::from_msgpack(zone_msgpack) { + Ok(z) => Some(z), + Err(e) => { + return self.response_error( + task, + ErrorCode::Internal { + detail: format!("array slice zone decode: {e}"), + }, + ); + } + } + }; + // The slice box and the zone prune together. The zone's transformer + // is built once for the whole scan. + let region = ArrayRegion::new(slice.clone(), zone); + let zone = region.zone.as_ref().map(PolygonZone::prepare); let schema = match self.array_engine.store(array_id) { Ok(store) => store.schema().clone(), @@ -127,13 +149,14 @@ impl CoreLoop { } }; - // Apply the shard bucket and slice coord filters. + // Apply the shard bucket, slice coord and zone filters. versions.retain(|(hp, coord, _sys_ms, _payload)| { if !shard_bucket_admits(shard_bucket, *hp) { return false; } // Check coord against slice predicate dim-by-dim. slice_coord_matches(coord, &slice) + && zone.as_ref().is_none_or(|z| z.contains(coord)) }); // Sort ascending by system_from_ms, ties by coord lexicographic order. @@ -204,8 +227,7 @@ impl CoreLoop { ); } }; - // The slice prunes whole prefixes by footer MBR before decode. - let region = ArrayRegion::new(slice.clone(), None); + // The region prunes whole prefixes by footer MBR before decode. let (mut resolved_tiles, truncated_before_horizon) = match store .scan_shard_tiles_at(cutoff, valid_at_ms, shard_bucket, Some(®ion)) { @@ -246,10 +268,16 @@ impl CoreLoop { if !shard_bucket_admits(shard_bucket, hp) { continue; } - if !tile_overlaps_slice(&sparse.mbr.dim_mins, &sparse.mbr.dim_maxs, &slice) { + if !region.tile_may_contain(&sparse.mbr) { continue; } - let filtered = match slice_sparse_owned(&schema, sparse, &slice) { + let sliced = slice_sparse_owned(&schema, sparse, &slice); + let filtered = match (sliced, zone.as_ref()) { + (Ok(t), None) => Ok(t), + (Ok(t), Some(z)) => filter_sparse(&schema, &t, |c| z.contains(c)), + (Err(e), _) => Err(e), + }; + let filtered = match filtered { Ok(t) => t, Err(e) => { return self.response_error( @@ -593,6 +621,7 @@ mod tests { shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, + zone_msgpack: Vec::new(), }); assert_eq!(r.status, Status::Ok, "slice failed: {r:?}"); let rows = decode_slice_rows(r.payload.as_ref()); @@ -642,6 +671,7 @@ mod tests { shard_bucket: None, system_time: nodedb_types::SystemTimeScope::AllVersions, valid_at_ms: None, + zone_msgpack: Vec::new(), }); assert_eq!(r.status, Status::Ok, "all-versions slice failed: {r:?}"); @@ -712,6 +742,7 @@ mod tests { shard_bucket: None, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, + zone_msgpack: Vec::new(), }); assert_eq!(r.status, Status::Ok, "slice+filter failed: {r:?}"); let rows = decode_slice_rows(r.payload.as_ref()); @@ -747,11 +778,129 @@ mod tests { shard_bucket: bucket, system_time: nodedb_types::SystemTimeScope::Current, valid_at_ms: None, + zone_msgpack: Vec::new(), }); assert_eq!(r.status, Status::Ok, "slice failed: {r:?}"); decode_slice_rows(r.payload.as_ref()) } + /// A pixel-mode schema over [`schema_2d_f64`]: 1-degree pixels from + /// (100°E, 8°N), x the column, y the row, in WGS 84. + fn georef_schema(name: &str) -> ArraySchema { + schema_2d_f64(name) + .with_georef(Some(nodedb_array::schema::ArrayGeoref { + srid: nodedb_types::geometry::Srid::Wgs84, + x_dim: 0, + y_dim: 1, + transform: Some(nodedb_array::schema::GeoTransform([ + 100.0, 1.0, 0.0, 8.0, 0.0, -1.0, + ])), + })) + .unwrap() + } + + /// A 4326 zone from 101°E to 103°E and 5°N to 7°N, framed over `schema`. + /// It holds the pixels with x in 1..=2 and y in 1..=2. + fn lon_lat_zone(schema: &ArraySchema) -> Vec { + use nodedb_array::query::region::ZonePolygon; + use nodedb_array::query::zone_frame::build_zone_frame; + let polygons = vec![ZonePolygon { + exterior: vec![ + [101.0, 5.0], + [103.0, 5.0], + [103.0, 7.0], + [101.0, 7.0], + [101.0, 5.0], + ], + holes: Vec::new(), + }]; + let frame = build_zone_frame(schema, nodedb_types::geometry::Srid::Wgs84, &polygons) + .expect("frame"); + zerompk::to_msgpack_vec(&PolygonZone { + x_dim: 0, + y_dim: 1, + polygons, + frame: Some(frame), + }) + .unwrap() + } + + fn sorted_coords(rows: &[serde_json::Map]) -> Vec<(i64, i64)> { + let mut coords: Vec<(i64, i64)> = rows.iter().map(row_coords).collect(); + coords.sort(); + coords + } + + #[test] + fn slice_with_a_zone_returns_only_the_cells_inside_it() { + let mut h = Harness::new(); + let s = georef_schema("t6_zone"); + let aid = ArrayId::new(TenantId::new(1), "t6_zone"); + h.open(&aid, &s, 0xD1); + let cells: Vec = (0..5) + .flat_map(|x| (0..5).map(move |y| cell(x, y, (x * 5 + y) as f64))) + .collect(); + h.put(&aid, cells, 1); + h.flush(&aid); + + let slice = ArraySlice::new(vec![None, None]); + let r = h.send(ArrayOp::Slice { + array_id: aid.clone(), + slice_msgpack: zerompk::to_msgpack_vec(&slice).unwrap(), + attr_projection: vec![], + limit: 0, + cell_filter: None, + shard_bucket: None, + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + zone_msgpack: lon_lat_zone(&s), + }); + assert_eq!(r.status, Status::Ok, "zone slice failed: {r:?}"); + let rows = decode_slice_rows(r.payload.as_ref()); + assert_eq!( + sorted_coords(&rows), + vec![(1, 1), (1, 2), (2, 1), (2, 2)], + "pixel centres 101.5..102.5°E, 5.5..6.5°N" + ); + } + + #[test] + fn slice_all_versions_with_a_zone_keeps_only_the_cells_inside_it() { + let mut h = Harness::new(); + let s = georef_schema("t6_zone_av"); + let aid = ArrayId::new(TenantId::new(1), "t6_zone_av"); + h.open(&aid, &s, 0xD2); + let at = |x: i64, y: i64, v: f64, sys: i64| ArrayPutCell { + system_from_ms: sys, + ..cell(x, y, v) + }; + h.put(&aid, vec![at(1, 1, 1.0, 100), at(4, 4, 9.0, 100)], 1); + h.flush(&aid); + h.put(&aid, vec![at(1, 1, 2.0, 200), at(4, 4, 8.0, 200)], 2); + h.flush(&aid); + + let slice = ArraySlice::new(vec![None, None]); + let r = h.send(ArrayOp::Slice { + array_id: aid.clone(), + slice_msgpack: zerompk::to_msgpack_vec(&slice).unwrap(), + attr_projection: vec![], + limit: 0, + cell_filter: None, + shard_bucket: None, + system_time: nodedb_types::SystemTimeScope::AllVersions, + valid_at_ms: None, + zone_msgpack: lon_lat_zone(&s), + }); + assert_eq!(r.status, Status::Ok, "zone audit slice failed: {r:?}"); + let rows = decode_slice_rows(r.payload.as_ref()); + let got: Vec<((i64, i64), f64)> = rows.iter().map(|r| (row_coords(r), row_v(r))).collect(); + assert_eq!( + got, + vec![((1, 1), 1.0), ((1, 1), 2.0)], + "both versions of the cell inside the zone, none of the cell outside it" + ); + } + fn row_coords(row: &serde_json::Map) -> (i64, i64) { ( row["x"].as_i64().expect("x coord"), diff --git a/nodedb/src/engine/array/engine.rs b/nodedb/src/engine/array/engine.rs index b5d817892..1f95e7c9d 100644 --- a/nodedb/src/engine/array/engine.rs +++ b/nodedb/src/engine/array/engine.rs @@ -105,8 +105,11 @@ impl ArrayEngine { schema: Arc, schema_hash: u64, ) -> ArrayEngineResult<()> { - if let Some(existing) = self.arrays.get(&id) { + if let Some(existing) = self.arrays.get_mut(&id) { if existing.schema_hash() == schema_hash { + // An equal hash means equal content. Only the georef can + // differ, so the open store adopts the new schema in place. + existing.adopt_schema(schema); return Ok(()); } return Err(ArrayEngineError::SchemaMismatch { @@ -329,6 +332,40 @@ mod tests { ); } + /// A reopen with the same hash adopts the new schema, which differs only + /// in its georef, and keeps the flushed segments readable. + #[test] + fn open_array_same_hash_adopts_a_georef_change() { + use crate::engine::array::test_support::put_one; + use nodedb_array::schema::{ArrayGeoref, GeoTransform}; + use tempfile::TempDir; + + let dir = TempDir::new().unwrap(); + let mut e = ArrayEngine::new(ArrayEngineConfig::new(dir.path().to_path_buf())).unwrap(); + e.open_array(aid(), schema(), 0xC0FFEE).unwrap(); + put_one(&mut e, 1, 1, 7, 1); + e.flush(&aid(), crate::types::replay_stamp::ReplayStamp::through(2)) + .unwrap(); + + let georef = ArrayGeoref { + srid: nodedb_types::geometry::Srid::Utm47N, + x_dim: 0, + y_dim: 1, + transform: Some(GeoTransform([600_000.0, 10.0, 0.0, 700_000.0, 0.0, -10.0])), + }; + let geo = Arc::new( + (*schema()) + .clone() + .with_georef(Some(georef.clone())) + .unwrap(), + ); + e.open_array(aid(), geo, 0xC0FFEE).unwrap(); + let store = e.store(&aid()).unwrap(); + assert_eq!(store.schema().georef, Some(georef)); + assert_eq!(store.manifest().segments.len(), 1); + assert_eq!(store.segments.len(), 1); + } + #[test] fn open_array_rejects_different_hash() { use tempfile::TempDir; diff --git a/nodedb/src/engine/array/store/catalog/store.rs b/nodedb/src/engine/array/store/catalog/store.rs index c160bca43..61cc718b7 100644 --- a/nodedb/src/engine/array/store/catalog/store.rs +++ b/nodedb/src/engine/array/store/catalog/store.rs @@ -113,6 +113,12 @@ impl ArrayStore { &self.schema } + /// Replace the schema of this open store. The caller passes a schema + /// with this store's `schema_hash`, so only its georef can differ. + pub fn adopt_schema(&mut self, schema: Arc) { + self.schema = schema; + } + pub fn schema_hash(&self) -> u64 { self.schema_hash } diff --git a/nodedb/tests/inproc/cases/surrogate_round_trip.rs b/nodedb/tests/inproc/cases/surrogate_round_trip.rs deleted file mode 100644 index 4b135b9bb..000000000 --- a/nodedb/tests/inproc/cases/surrogate_round_trip.rs +++ /dev/null @@ -1,763 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Single-node integration test for surrogate identity round-trip. -//! -//! Asserts three properties: -//! -//! 1. **Per-engine uniqueness** — the surrogate values assigned to rows in -//! each engine are distinct from one another within the engine's own -//! insert batch (no engine re-uses a surrogate for two different rows). -//! -//! 2. **Global uniqueness** — all surrogates across every engine form a -//! set with no duplicates. -//! -//! 3. **Cross-engine bitmap intersection** — a three-way intersection of -//! `SurrogateBitmap`s produced from Vector, Graph, and Array engine -//! inserts yields the correct set, and actually running queries with -//! the intersected bitmap as `filter_bitmap` / `frontier_bitmap` / -//! `cell_filter` returns only rows whose surrogate is in the -//! intersection. -//! -//! The test drives a single `CoreLoop` via the ring-buffer harness (the -//! same pattern as `cross_engine_bitmap_currency.rs`) so no server or -//! network is required. - -use std::collections::HashSet; -use std::time::{Duration, Instant}; - -use nodedb::bridge::dispatch::{BridgeRequest, BridgeResponse}; -use nodedb::bridge::envelope::{Priority, Request, Status}; -use nodedb::data::executor::core_loop::CoreLoop; -use nodedb::data::executor::response_codec::decode_payload_to_json; -use nodedb::engine::array::wal::ArrayPutCell; -use nodedb::engine::graph::edge_store::Direction; -use nodedb::types::*; -use nodedb_array::query::slice::Slice as ArraySlice; -use nodedb_array::schema::ArraySchemaBuilder; -use nodedb_array::schema::attr_spec::{AttrSpec, AttrType}; -use nodedb_array::schema::dim_spec::{DimSpec, DimType}; -use nodedb_array::types::ArrayId; -use nodedb_array::types::cell_value::value::CellValue; -use nodedb_array::types::coord::value::CoordValue; -use nodedb_array::types::domain::{Domain, DomainBound}; -use nodedb_bridge::buffer::{Consumer, Producer, RingBuffer}; -use nodedb_physical::physical_plan::{ - ArrayOp, ColumnarInsertIntent, ColumnarOp, DocumentOp, GraphOp, KvOp, PhysicalPlan, VectorOp, -}; -use nodedb_types::vector_distance::DistanceMetric; -use nodedb_types::{Surrogate, SurrogateBitmap}; - -// ── Harness ──────────────────────────────────────────────────────────────── - -fn open_core() -> ( - CoreLoop, - Producer, - Consumer, - tempfile::TempDir, -) { - let dir = tempfile::tempdir().unwrap(); - let (req_tx, req_rx) = RingBuffer::channel::(128); - let (resp_tx, resp_rx) = RingBuffer::channel::(128); - let core = CoreLoop::open( - 0, - req_rx, - resp_tx, - dir.path(), - std::sync::Arc::new(nodedb_types::OrdinalClock::new()), - nodedb::data::executor::core_loop::test_governor(), - ) - .unwrap(); - (core, req_tx, resp_rx, dir) -} - -fn make_req(plan: PhysicalPlan) -> Request { - Request { - request_id: RequestId::new(1), - tenant_id: TenantId::new(1), - vshard_id: VShardId::new(0), - database_id: nodedb::types::DatabaseId::DEFAULT, - plan, - deadline: Instant::now() + Duration::from_secs(5), - priority: Priority::Normal, - trace_id: nodedb_types::TraceId::ZERO, - consistency: ReadConsistency::Strong, - idempotency_key: None, - event_source: nodedb::event::EventSource::User, - user_roles: Vec::new(), - user_id: None, - statement_digest: None, - txn_id: None, - wal_lsn: None, - resolved_now_ms: None, - commit_hlc: None, - entry_version: None, - admission: nodedb::bridge::envelope::Admission::Admitted, - } -} - -/// Push a plan, tick, pop response, assert `Ok`. Returns the payload. -fn send_ok( - core: &mut CoreLoop, - tx: &mut Producer, - rx: &mut Consumer, - plan: PhysicalPlan, -) -> Vec { - tx.try_push(BridgeRequest::unfloored(make_req(plan))) - .unwrap(); - core.tick(); - let resp = rx.try_pop().unwrap(); - assert_eq!( - resp.inner.status, - Status::Ok, - "expected Ok status, got {:?}: {:?}", - resp.inner.status, - resp.inner.error_code - ); - resp.inner.payload.to_vec() -} - -/// Batch-push plans without consuming responses in between, then drain -/// all responses at once. Each plan must succeed. -fn send_batch_ok( - core: &mut CoreLoop, - tx: &mut Producer, - rx: &mut Consumer, - plans: Vec, -) { - let n = plans.len(); - for plan in plans { - tx.try_push(BridgeRequest::unfloored(make_req(plan))) - .unwrap(); - } - core.tick(); - for _ in 0..n { - let resp = rx.try_pop().unwrap(); - assert_eq!( - resp.inner.status, - Status::Ok, - "batch send: expected Ok, got {:?}: {:?}", - resp.inner.status, - resp.inner.error_code - ); - } -} - -fn parse_json(payload: &[u8]) -> serde_json::Value { - let s = decode_payload_to_json(payload); - serde_json::from_str(&s).unwrap_or(serde_json::Value::Array(vec![])) -} - -fn extract_vector_surrogates(payload: &[u8]) -> Vec { - parse_json(payload) - .as_array() - .unwrap_or(&vec![]) - .iter() - .filter_map(|h| h.get("id").and_then(|v| v.as_str())) - .filter_map(|key| nodedb_types::StorageKey::parse(key).map(|k| k.surrogate().as_u32())) - .collect() -} - -// ── Surrogate constants ──────────────────────────────────────────────────── -// -// We partition the 32-bit surrogate space into non-overlapping ranges so -// uniqueness across engines is structurally guaranteed (no accidental reuse). -// Then we select a deliberate overlap set for the three-way intersection test. -// -// Layout: -// Vector : 10, 11, 12, 13 -// Document: 20, 21, 22 -// Graph : 10, 12, 30, 31 ← 10 and 12 shared with Vector -// KV : 40, 41 -// Columnar: 50, 51, 52 -// Array : 10, 13, 60, 61 ← 10 and 13 shared with Vector; 12 NOT shared -// FTS : 70, 71 (written as Document because TextOp has no insert op; -// the doc engine drives the BM25 index) -// Spatial : 80, 81 (written as Document because SpatialOp has no insert) -// -// Three-way intersection (Vector ∩ Graph ∩ Array) = {10} -// - surrogate 10 is in ALL THREE: vector row, graph edge endpoints, array cell -// - surrogate 11 is Vector-only -// - surrogate 12 is Vector + Graph (not Array) -// - surrogate 13 is Vector + Array (not Graph) -// - surrogates 30,31 are Graph-only -// - surrogates 60,61 are Array-only - -const V_SURS: &[u32] = &[10, 11, 12, 13]; // Vector -const D_SURS: &[u32] = &[20, 21, 22]; // Document schemaless -const G_SRC_SURS: &[u32] = &[10, 12, 30, 31]; // Graph src nodes -const KV_SURS: &[u32] = &[40, 41]; // KV -const COL_SURS: &[u32] = &[50, 51, 52]; // Columnar -const ARR_SURS: &[u32] = &[10, 13, 60, 61]; // Array cells -const FTS_SURS: &[u32] = &[70, 71]; // FTS (via Document) -const SPATIAL_SURS: &[u32] = &[80, 81]; // Spatial (via Document + surrogate field) - -// ── Helper: build a simple 2-D array schema ────────────────────────────── - -fn array_schema_1d(name: &str) -> nodedb_array::schema::ArraySchema { - ArraySchemaBuilder::new(name) - .dim(DimSpec::new( - "x", - DimType::Int64, - Domain::new(DomainBound::Int64(0), DomainBound::Int64(255)), - )) - .attr(AttrSpec::new("v", AttrType::Float64, true)) - .tile_extents(vec![32]) - .build() - .unwrap() -} - -// ── Test ─────────────────────────────────────────────────────────────────── - -/// Main assertion: insert into eight engine paths, check uniqueness, check -/// the three-way Vector ∩ Graph ∩ Array bitmap intersection via real query -/// execution. -#[test] -fn surrogate_round_trip_all_engines() { - let (mut core, mut tx, mut rx, _dir) = open_core(); - - // ── 1. Vector inserts ─────────────────────────────────────────────── - // Each vector gets a distinct embedding so all appear near [1,0,0]. - let vec_plans: Vec = V_SURS - .iter() - .copied() - .map(|s| { - PhysicalPlan::Vector(VectorOp::Insert { - collection: nodedb_types::QualifiedCollection::new( - nodedb_types::DatabaseId::DEFAULT, - "vec_col", - ), - vector: vec![1.0, s as f32 * 0.01, 0.0], - dim: 3, - field_name: String::new(), - surrogate: Surrogate::new(s), - pk_bytes: None, - provenance: None, - }) - }) - .collect(); - send_batch_ok(&mut core, &mut tx, &mut rx, vec_plans); - - // ── 2. Document (schemaless) inserts ──────────────────────────────── - for &s in D_SURS { - let hex = format!("{s:08x}"); - let doc = serde_json::json!({ "id": hex, "kind": "doc" }); - send_ok( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Document(DocumentOp::PointPut { - collection: nodedb_types::QualifiedCollection::new( - nodedb_types::DatabaseId::DEFAULT, - "doc_col", - ), - document_id: hex.clone(), - value: serde_json::to_vec(&doc).unwrap(), - surrogate: Surrogate::new(s), - pk_bytes: hex.into_bytes(), - returning: None, - rls_filters: Vec::new(), - resolved_sum_targets: Vec::new(), - }), - ); - } - - // ── 3. Graph edge inserts ──────────────────────────────────────────── - // Each src node gets a surrogate in G_SRC_SURS; dst node uses s+100. - for &s in G_SRC_SURS { - let src = format!("n{s}"); - let dst = format!("n{}", s + 100); - send_ok( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Graph(GraphOp::EdgePut { - collection: nodedb_types::QualifiedCollection::new( - nodedb_types::DatabaseId::DEFAULT, - "graph_col", - ), - src_id: src.clone(), - label: "LINKED".into(), - dst_id: dst.clone(), - properties: vec![], - src_surrogate: Surrogate::new(s), - dst_surrogate: Surrogate::new(s + 100), - }), - ); - } - - // ── 4. KV inserts ──────────────────────────────────────────────────── - for &s in KV_SURS { - let key = format!("k{s}").into_bytes(); - // Minimal Binary Tuple value (empty). - send_ok( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Kv(KvOp::Put { - collection: nodedb_types::QualifiedCollection::new( - nodedb_types::DatabaseId::DEFAULT, - "kv_col", - ), - key: key.clone(), - value: vec![], - ttl_ms: 0, - surrogate: Surrogate::new(s), - returning: None, - rls_filters: Vec::new(), - provenance: None, - }), - ); - } - - // ── 5. Columnar inserts ────────────────────────────────────────────── - { - let rows_json = serde_json::json!( - COL_SURS - .iter() - .map(|&s| serde_json::json!({ "id": format!("{s:08x}"), "val": s })) - .collect::>() - ); - let rows_mp = nodedb_types::json_to_msgpack(&rows_json).unwrap(); - send_ok( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Columnar(ColumnarOp::Insert { - collection: nodedb_types::QualifiedCollection::new( - nodedb_types::DatabaseId::DEFAULT, - "col_col", - ), - payload: rows_mp, - format: "msgpack".into(), - intent: ColumnarInsertIntent::Insert, - on_conflict_updates: Vec::new(), - surrogates: COL_SURS.iter().copied().map(Surrogate::new).collect(), - schema_bytes: Vec::new(), - provenance: None, - wal_lsn: None, - rls_write_check: nodedb_types::RlsWriteCheck::NoPolicyApplies, - returning: None, - rls_filters: Vec::new(), - }), - ); - } - - // ── 6. Array inserts ───────────────────────────────────────────────── - { - let aid = ArrayId::new(TenantId::new(1), "arr_col"); - let schema = array_schema_1d("arr_col"); - let schema_bytes = zerompk::to_msgpack_vec(&schema).unwrap(); - - send_ok( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Array(ArrayOp::OpenArray { - array_id: aid.clone(), - schema_msgpack: schema_bytes, - schema_hash: 1, - prefix_bits: 8, - audit_retain_ms: None, - minimum_audit_retain_ms: None, - }), - ); - - let cells: Vec = ARR_SURS - .iter() - .copied() - .enumerate() - .map(|(i, s)| ArrayPutCell { - coord: vec![CoordValue::Int64(i as i64)], - attrs: vec![CellValue::Float64(s as f64)], - surrogate: Surrogate::new(s), - system_from_ms: 0, - valid_from_ms: 0, - valid_until_ms: i64::MAX, - }) - .collect(); - let cells_mp = zerompk::to_msgpack_vec(&cells).unwrap(); - - send_ok( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Array(ArrayOp::Put { - array_id: aid.clone(), - cells_msgpack: cells_mp, - wal_lsn: 1, - provenance: None, - vshard_id: 0, - }), - ); - } - - // ── 7. FTS inserts (via Document engine; text is indexed separately) ─ - for &s in FTS_SURS { - let hex = format!("{s:08x}"); - let doc = serde_json::json!({ "id": hex, "body": format!("learning keyword {s}") }); - send_ok( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Document(DocumentOp::PointPut { - collection: nodedb_types::QualifiedCollection::new( - nodedb_types::DatabaseId::DEFAULT, - "fts_col", - ), - document_id: hex.clone(), - value: serde_json::to_vec(&doc).unwrap(), - surrogate: Surrogate::new(s), - pk_bytes: hex.into_bytes(), - returning: None, - rls_filters: Vec::new(), - resolved_sum_targets: Vec::new(), - }), - ); - } - - // ── 8. Spatial inserts (via Document engine; geometry stored as field) ─ - for &s in SPATIAL_SURS { - let hex = format!("{s:08x}"); - let lon = s as f64 * 0.01; - let doc = serde_json::json!({ "id": hex, "geom": { "type": "Point", "coordinates": [lon, 0.0] } }); - send_ok( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Document(DocumentOp::PointPut { - collection: nodedb_types::QualifiedCollection::new( - nodedb_types::DatabaseId::DEFAULT, - "spatial_col", - ), - document_id: hex.clone(), - value: serde_json::to_vec(&doc).unwrap(), - surrogate: Surrogate::new(s), - pk_bytes: hex.into_bytes(), - returning: None, - rls_filters: Vec::new(), - resolved_sum_targets: Vec::new(), - }), - ); - } - - // ══════════════════════════════════════════════════════════════════════ - // Assertion A: Global surrogate uniqueness across all engines. - // - // We control all surrogate values; collect every value and assert the - // multi-set has no duplicates. The non-overlap was designed into the - // constants above; this assertion catches any future constant change - // that accidentally introduces a collision. - // ══════════════════════════════════════════════════════════════════════ - let all_surs: Vec = [ - V_SURS, - D_SURS, - G_SRC_SURS, - KV_SURS, - COL_SURS, - ARR_SURS, - FTS_SURS, - SPATIAL_SURS, - ] - .iter() - .flat_map(|s| s.iter().copied()) - .collect(); - - // Surrogates 10, 12, 13 intentionally appear in multiple engines (the - // cross-engine join scenario). The uniqueness property we assert here - // is that each engine's OWN insert batch uses distinct values — not - // that the same physical surrogate never appears in two engines (that - // would prohibit the cross-engine join design). - for slice in [ - V_SURS, - D_SURS, - G_SRC_SURS, - KV_SURS, - COL_SURS, - ARR_SURS, - FTS_SURS, - SPATIAL_SURS, - ] { - let set: HashSet = slice.iter().copied().collect(); - assert_eq!( - set.len(), - slice.len(), - "engine surrogate batch must contain no duplicates: {slice:?}" - ); - } - - // Build the global set (deduped) and assert no same surrogate maps to - // two different rows in the SAME engine. - let global_set: HashSet = all_surs.iter().copied().collect(); - // Global set has fewer elements than all_surs because of intentional - // cross-engine reuse of {10, 12, 13}. What we assert is that within - // each engine batch, every surrogate is unique. (Validated above.) - assert!( - global_set.contains(&10), - "surrogate 10 must be in the global set" - ); - - // ══════════════════════════════════════════════════════════════════════ - // Assertion B: Cross-engine bitmap intersection (Vector ∩ Graph ∩ Array). - // - // Design: - // bitmap_v = {10, 11, 12, 13} (all Vector surrogates) - // bitmap_g = {10, 12, 30, 31} (all Graph src surrogates) - // bitmap_a = {10, 13, 60, 61} (all Array cell surrogates) - // intersection = bitmap_v ∩ bitmap_g ∩ bitmap_a = {10} - // - // For each engine we run a query with `intersection` as the prefilter - // and assert only surrogate 10 is returned. - // ══════════════════════════════════════════════════════════════════════ - - let bitmap_v: SurrogateBitmap = V_SURS.iter().copied().map(Surrogate::new).collect(); - let bitmap_g: SurrogateBitmap = G_SRC_SURS.iter().copied().map(Surrogate::new).collect(); - let bitmap_a: SurrogateBitmap = ARR_SURS.iter().copied().map(Surrogate::new).collect(); - - let intersection = bitmap_v.intersect(&bitmap_g).intersect(&bitmap_a); - - assert_eq!( - intersection.len(), - 1, - "three-way intersection must contain exactly surrogate 10" - ); - assert!( - intersection.contains(Surrogate::new(10)), - "surrogate 10 must be in the intersection" - ); - - // B1 — Vector search with intersection as filter_bitmap. - // Unfiltered search returns all four Vector surrogates (11, 12, 13 - // also match the query vector near [1,0,0]). - let all_vec = send_ok( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Vector(VectorOp::Search { - collection: nodedb_types::QualifiedCollection::new( - nodedb_types::DatabaseId::DEFAULT, - "vec_col", - ), - query_vector: vec![1.0f32, 0.0, 0.0], - top_k: 20, - ef_search: 0, - filter_bitmap: None, - field_name: String::new(), - rls_filters: Vec::new(), - inline_prefilter_plan: None, - ann_options: Default::default(), - skip_payload_fetch: false, - payload_filters: Vec::new(), - metric: DistanceMetric::L2, - }), - ); - let all_vec_surs = extract_vector_surrogates(&all_vec); - assert!( - all_vec_surs.len() >= 2, - "unfiltered vector search should return at least 2 results, got {all_vec_surs:?}" - ); - - let filtered_vec = send_ok( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Vector(VectorOp::Search { - collection: nodedb_types::QualifiedCollection::new( - nodedb_types::DatabaseId::DEFAULT, - "vec_col", - ), - query_vector: vec![1.0f32, 0.0, 0.0], - top_k: 20, - ef_search: 0, - filter_bitmap: Some(intersection.clone()), - field_name: String::new(), - rls_filters: Vec::new(), - inline_prefilter_plan: None, - ann_options: Default::default(), - skip_payload_fetch: false, - payload_filters: Vec::new(), - metric: DistanceMetric::L2, - }), - ); - let filtered_vec_surs = extract_vector_surrogates(&filtered_vec); - - // Every returned surrogate must be in the intersection. - for &s in &filtered_vec_surs { - assert!( - intersection.contains(Surrogate::new(s)), - "vector search returned surrogate {s} not in intersection" - ); - } - // Surrogates 11, 12, 13 must not appear (not in intersection). - for &absent in &[11u32, 12, 13] { - assert!( - !filtered_vec_surs.contains(&absent), - "surrogate {absent} is Vector-only or Vector+one-engine; must be absent from intersection-filtered search" - ); - } - - // B2 — Graph hop with frontier_bitmap = intersection. - // Insert an edge from n10 → n11 so n11 is reachable. - // With frontier_bitmap = {10}, the hop should not traverse to n11 - // (surrogate 11 is not in the intersection). - send_ok( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Graph(GraphOp::EdgePut { - collection: nodedb_types::QualifiedCollection::new( - nodedb_types::DatabaseId::DEFAULT, - "graph_col", - ), - src_id: "n10".into(), - label: "REACH".into(), - dst_id: "n11".into(), - properties: vec![], - src_surrogate: Surrogate::new(10), - dst_surrogate: Surrogate::new(11), - }), - ); - - let hop_all = send_ok( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Graph(GraphOp::Hop { - start_nodes: vec!["n10".into()], - edge_labels: vec!["REACH".into()], - direction: Direction::Out, - depth: 1, - options: Default::default(), - rls_filters: Vec::new(), - frontier_bitmap: None, - collection: None, - }), - ); - let hop_all_json = decode_payload_to_json(&hop_all); - // Unfiltered: n11 is reachable from n10 via REACH. - assert!( - hop_all_json.contains("n11"), - "unfiltered hop from n10 must reach n11: {hop_all_json}" - ); - - let hop_filtered = send_ok( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Graph(GraphOp::Hop { - start_nodes: vec!["n10".into()], - edge_labels: vec!["REACH".into()], - direction: Direction::Out, - depth: 1, - options: Default::default(), - rls_filters: Vec::new(), - frontier_bitmap: Some(intersection.clone()), - collection: None, - }), - ); - let hop_filtered_json = decode_payload_to_json(&hop_filtered); - // Filtered: n11's surrogate (11) is not in the intersection — hop must - // not include it. - assert!( - !hop_filtered_json.contains("n11"), - "intersection-filtered hop must not reach n11 (surrogate 11 not in intersection): \ - {hop_filtered_json}" - ); - - // B3 — Array slice with cell_filter = intersection. - { - let aid = ArrayId::new(TenantId::new(1), "arr_col"); - // Full slice (no filter). - // Full slice: no restrictions on any dimension (None = unbounded). - let full_slice = ArraySlice::new(vec![None]); - let slice_mp = zerompk::to_msgpack_vec(&full_slice).unwrap(); - - let slice_all = send_ok( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Array(ArrayOp::Slice { - array_id: aid.clone(), - slice_msgpack: slice_mp.clone(), - attr_projection: vec![], - limit: 100, - cell_filter: None, - shard_bucket: None, - system_time: nodedb_types::SystemTimeScope::Current, - valid_at_ms: None, - }), - ); - // Unfiltered: should see all ARR_SURS cells. - let slice_all_json = decode_payload_to_json(&slice_all); - assert!( - !slice_all_json.is_empty(), - "unfiltered array slice must return rows" - ); - - let slice_filtered = send_ok( - &mut core, - &mut tx, - &mut rx, - PhysicalPlan::Array(ArrayOp::Slice { - array_id: aid.clone(), - slice_msgpack: slice_mp, - attr_projection: vec![], - limit: 100, - cell_filter: Some(intersection.clone()), - shard_bucket: None, - system_time: nodedb_types::SystemTimeScope::Current, - valid_at_ms: None, - }), - ); - // Filtered: intersection ∩ ARR_SURS = {10}. Surrogate 10 is at - // coord [0] with attr value 10.0 (see ARR_SURS insert loop above). - // Surrogates 13 (coord [1]), 60 (coord [2]), 61 (coord [3]) must not - // appear. The response is zerompk-encoded ArrayCell values; we - // transcode to JSON for inspection. - // Local DP slice responses are wrapped as `ArraySliceResponse { rows_msgpack: - // Vec, truncated_before_horizon: bool }`. The inner `rows_msgpack` is a - // separate msgpack array of `ArrayCell` values. - #[derive(serde::Deserialize, zerompk::FromMessagePack)] - #[msgpack(map)] - struct SliceEnvelope { - rows_msgpack: Vec, - #[allow(dead_code)] - truncated_before_horizon: bool, - } - fn unwrap_rows(payload: &[u8]) -> serde_json::Value { - let envelope: SliceEnvelope = match zerompk::from_msgpack(payload) { - Ok(e) => e, - Err(_) => return serde_json::Value::Array(vec![]), - }; - let json_str = - nodedb_types::msgpack_to_json_string(&envelope.rows_msgpack).unwrap_or_default(); - serde_json::from_str(&json_str).unwrap_or(serde_json::Value::Array(vec![])) - } - let filtered_cells = unwrap_rows(&slice_filtered); - let unfiltered_cells = unwrap_rows(&slice_all); - assert_eq!( - unfiltered_cells.as_array().map(|a| a.len()).unwrap_or(0), - ARR_SURS.len(), - "unfiltered slice must return all {} array cells", - ARR_SURS.len() - ); - - let filtered_count = filtered_cells.as_array().map(|a| a.len()).unwrap_or(0); - assert_eq!( - filtered_count, 1, - "intersection-filtered slice must return exactly 1 cell (surrogate 10), \ - got {filtered_count}: {filtered_cells}" - ); - - // The surviving cell must have attr value 10.0 (the value we stored at - // coord [0] which received surrogate 10). A slice row names each attr. - let cell = filtered_cells - .as_array() - .and_then(|a| a.first()) - .expect("one filtered cell"); - assert_eq!( - cell.get("v").and_then(|v| v.as_f64()), - Some(10.0), - "filtered cell attr `v` must be 10.0 (surrogate-10's stored value): {cell}" - ); - } -} diff --git a/nodedb/tests/inproc/cases/surrogate_round_trip/harness.rs b/nodedb/tests/inproc/cases/surrogate_round_trip/harness.rs new file mode 100644 index 000000000..444207de0 --- /dev/null +++ b/nodedb/tests/inproc/cases/surrogate_round_trip/harness.rs @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: BUSL-1.1 + +use std::time::{Duration, Instant}; + +use nodedb::bridge::dispatch::{BridgeRequest, BridgeResponse}; +use nodedb::bridge::envelope::{Priority, Request, Status}; +use nodedb::data::executor::core_loop::CoreLoop; +use nodedb::data::executor::response_codec::decode_payload_to_json; +use nodedb::types::*; +use nodedb_array::schema::ArraySchemaBuilder; +use nodedb_array::schema::attr_spec::{AttrSpec, AttrType}; +use nodedb_array::schema::dim_spec::{DimSpec, DimType}; +use nodedb_array::types::domain::{Domain, DomainBound}; +use nodedb_bridge::buffer::{Consumer, Producer, RingBuffer}; +use nodedb_physical::physical_plan::PhysicalPlan; + +// ── Harness ──────────────────────────────────────────────────────────────── + +pub(super) fn open_core() -> ( + CoreLoop, + Producer, + Consumer, + tempfile::TempDir, +) { + let dir = tempfile::tempdir().unwrap(); + let (req_tx, req_rx) = RingBuffer::channel::(128); + let (resp_tx, resp_rx) = RingBuffer::channel::(128); + let core = CoreLoop::open( + 0, + req_rx, + resp_tx, + dir.path(), + std::sync::Arc::new(nodedb_types::OrdinalClock::new()), + nodedb::data::executor::core_loop::test_governor(), + ) + .unwrap(); + (core, req_tx, resp_rx, dir) +} + +pub(super) fn make_req(plan: PhysicalPlan) -> Request { + Request { + request_id: RequestId::new(1), + tenant_id: TenantId::new(1), + vshard_id: VShardId::new(0), + database_id: nodedb::types::DatabaseId::DEFAULT, + plan, + deadline: Instant::now() + Duration::from_secs(5), + priority: Priority::Normal, + trace_id: nodedb_types::TraceId::ZERO, + consistency: ReadConsistency::Strong, + idempotency_key: None, + event_source: nodedb::event::EventSource::User, + user_roles: Vec::new(), + user_id: None, + statement_digest: None, + txn_id: None, + wal_lsn: None, + resolved_now_ms: None, + commit_hlc: None, + entry_version: None, + admission: nodedb::bridge::envelope::Admission::Admitted, + } +} + +/// Push a plan, tick, pop response, assert `Ok`. Returns the payload. +pub(super) fn send_ok( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, + plan: PhysicalPlan, +) -> Vec { + tx.try_push(BridgeRequest::unfloored(make_req(plan))) + .unwrap(); + core.tick(); + let resp = rx.try_pop().unwrap(); + assert_eq!( + resp.inner.status, + Status::Ok, + "expected Ok status, got {:?}: {:?}", + resp.inner.status, + resp.inner.error_code + ); + resp.inner.payload.to_vec() +} + +/// Batch-push plans without consuming responses in between, then drain +/// all responses at once. Each plan must succeed. +pub(super) fn send_batch_ok( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, + plans: Vec, +) { + let n = plans.len(); + for plan in plans { + tx.try_push(BridgeRequest::unfloored(make_req(plan))) + .unwrap(); + } + core.tick(); + for _ in 0..n { + let resp = rx.try_pop().unwrap(); + assert_eq!( + resp.inner.status, + Status::Ok, + "batch send: expected Ok, got {:?}: {:?}", + resp.inner.status, + resp.inner.error_code + ); + } +} + +pub(super) fn parse_json(payload: &[u8]) -> serde_json::Value { + let s = decode_payload_to_json(payload); + serde_json::from_str(&s).unwrap_or(serde_json::Value::Array(vec![])) +} + +pub(super) fn extract_vector_surrogates(payload: &[u8]) -> Vec { + parse_json(payload) + .as_array() + .unwrap_or(&vec![]) + .iter() + .filter_map(|h| h.get("id").and_then(|v| v.as_str())) + .filter_map(|key| nodedb_types::StorageKey::parse(key).map(|k| k.surrogate().as_u32())) + .collect() +} + +// ── Surrogate constants ──────────────────────────────────────────────────── +// +// The 32-bit surrogate space is partitioned into non-overlapping ranges so +// uniqueness across engines is structurally guaranteed (no accidental reuse). +// A deliberate overlap set drives the three-way intersection test. +// +// Layout: +// Vector : 10, 11, 12, 13 +// Document: 20, 21, 22 +// Graph : 10, 12, 30, 31 ← 10 and 12 shared with Vector +// KV : 40, 41 +// Columnar: 50, 51, 52 +// Array : 10, 13, 60, 61 ← 10 and 13 shared with Vector; 12 NOT shared +// FTS : 70, 71 (written as Document because TextOp has no insert op; +// the doc engine drives the BM25 index) +// Spatial : 80, 81 (written as Document because SpatialOp has no insert) +// +// Three-way intersection (Vector ∩ Graph ∩ Array) = {10} +// - surrogate 10 is in ALL THREE: vector row, graph edge endpoints, array cell +// - surrogate 11 is Vector-only +// - surrogate 12 is Vector + Graph (not Array) +// - surrogate 13 is Vector + Array (not Graph) +// - surrogates 30,31 are Graph-only +// - surrogates 60,61 are Array-only + +pub(super) const V_SURS: &[u32] = &[10, 11, 12, 13]; // Vector +pub(super) const D_SURS: &[u32] = &[20, 21, 22]; // Document schemaless +pub(super) const G_SRC_SURS: &[u32] = &[10, 12, 30, 31]; // Graph src nodes +pub(super) const KV_SURS: &[u32] = &[40, 41]; // KV +pub(super) const COL_SURS: &[u32] = &[50, 51, 52]; // Columnar +pub(super) const ARR_SURS: &[u32] = &[10, 13, 60, 61]; // Array cells +pub(super) const FTS_SURS: &[u32] = &[70, 71]; // FTS (via Document) +pub(super) const SPATIAL_SURS: &[u32] = &[80, 81]; // Spatial (via Document + surrogate field) + +// ── Helper: build a simple 1-D array schema ────────────────────────────── + +pub(super) fn array_schema_1d(name: &str) -> nodedb_array::schema::ArraySchema { + ArraySchemaBuilder::new(name) + .dim(DimSpec::new( + "x", + DimType::Int64, + Domain::new(DomainBound::Int64(0), DomainBound::Int64(255)), + )) + .attr(AttrSpec::new("v", AttrType::Float64, true)) + .tile_extents(vec![32]) + .build() + .unwrap() +} diff --git a/nodedb/tests/inproc/cases/surrogate_round_trip/inserts.rs b/nodedb/tests/inproc/cases/surrogate_round_trip/inserts.rs new file mode 100644 index 000000000..93395c1b1 --- /dev/null +++ b/nodedb/tests/inproc/cases/surrogate_round_trip/inserts.rs @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: BUSL-1.1 + +use nodedb::bridge::dispatch::{BridgeRequest, BridgeResponse}; +use nodedb::data::executor::core_loop::CoreLoop; +use nodedb::engine::array::wal::ArrayPutCell; +use nodedb::types::TenantId; +use nodedb_array::types::ArrayId; +use nodedb_array::types::cell_value::value::CellValue; +use nodedb_array::types::coord::value::CoordValue; +use nodedb_bridge::buffer::{Consumer, Producer}; +use nodedb_physical::physical_plan::{ + ArrayOp, ColumnarInsertIntent, ColumnarOp, DocumentOp, GraphOp, KvOp, PhysicalPlan, VectorOp, +}; +use nodedb_types::Surrogate; + +use super::harness::{ + ARR_SURS, COL_SURS, D_SURS, FTS_SURS, G_SRC_SURS, KV_SURS, SPATIAL_SURS, V_SURS, + array_schema_1d, send_batch_ok, send_ok, +}; + +// ── 1. Vector inserts ─────────────────────────────────────────────── +pub(super) fn insert_vector( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, +) { + // Each vector gets a distinct embedding so all appear near [1,0,0]. + let vec_plans: Vec = V_SURS + .iter() + .copied() + .map(|s| { + PhysicalPlan::Vector(VectorOp::Insert { + collection: nodedb_types::QualifiedCollection::new( + nodedb_types::DatabaseId::DEFAULT, + "vec_col", + ), + vector: vec![1.0, s as f32 * 0.01, 0.0], + dim: 3, + field_name: String::new(), + surrogate: Surrogate::new(s), + pk_bytes: None, + provenance: None, + }) + }) + .collect(); + send_batch_ok(core, tx, rx, vec_plans); +} + +// ── 2. Document (schemaless) inserts ──────────────────────────────── +pub(super) fn insert_document( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, +) { + for &s in D_SURS { + let hex = format!("{s:08x}"); + let doc = serde_json::json!({ "id": hex, "kind": "doc" }); + send_ok( + core, + tx, + rx, + PhysicalPlan::Document(DocumentOp::PointPut { + collection: nodedb_types::QualifiedCollection::new( + nodedb_types::DatabaseId::DEFAULT, + "doc_col", + ), + document_id: hex.clone(), + value: serde_json::to_vec(&doc).unwrap(), + surrogate: Surrogate::new(s), + pk_bytes: hex.into_bytes(), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + }), + ); + } +} + +// ── 3. Graph edge inserts ──────────────────────────────────────────── +// Each src node gets a surrogate in G_SRC_SURS; dst node uses s+100. +pub(super) fn insert_graph( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, +) { + for &s in G_SRC_SURS { + let src = format!("n{s}"); + let dst = format!("n{}", s + 100); + send_ok( + core, + tx, + rx, + PhysicalPlan::Graph(GraphOp::EdgePut { + collection: nodedb_types::QualifiedCollection::new( + nodedb_types::DatabaseId::DEFAULT, + "graph_col", + ), + src_id: src.clone(), + label: "LINKED".into(), + dst_id: dst.clone(), + properties: vec![], + src_surrogate: Surrogate::new(s), + dst_surrogate: Surrogate::new(s + 100), + }), + ); + } +} + +// ── 4. KV inserts ──────────────────────────────────────────────────── +pub(super) fn insert_kv( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, +) { + for &s in KV_SURS { + let key = format!("k{s}").into_bytes(); + // Minimal Binary Tuple value (empty). + send_ok( + core, + tx, + rx, + PhysicalPlan::Kv(KvOp::Put { + collection: nodedb_types::QualifiedCollection::new( + nodedb_types::DatabaseId::DEFAULT, + "kv_col", + ), + key: key.clone(), + value: vec![], + ttl_ms: 0, + surrogate: Surrogate::new(s), + returning: None, + rls_filters: Vec::new(), + provenance: None, + }), + ); + } +} + +// ── 5. Columnar inserts ────────────────────────────────────────────── +pub(super) fn insert_columnar( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, +) { + let rows_json = serde_json::json!( + COL_SURS + .iter() + .map(|&s| serde_json::json!({ "id": format!("{s:08x}"), "val": s })) + .collect::>() + ); + let rows_mp = nodedb_types::json_to_msgpack(&rows_json).unwrap(); + send_ok( + core, + tx, + rx, + PhysicalPlan::Columnar(ColumnarOp::Insert { + collection: nodedb_types::QualifiedCollection::new( + nodedb_types::DatabaseId::DEFAULT, + "col_col", + ), + payload: rows_mp, + format: "msgpack".into(), + intent: ColumnarInsertIntent::Insert, + on_conflict_updates: Vec::new(), + surrogates: COL_SURS.iter().copied().map(Surrogate::new).collect(), + schema_bytes: Vec::new(), + provenance: None, + wal_lsn: None, + rls_write_check: nodedb_types::RlsWriteCheck::NoPolicyApplies, + returning: None, + rls_filters: Vec::new(), + }), + ); +} + +// ── 6. Array inserts ───────────────────────────────────────────────── +pub(super) fn insert_array( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, +) { + let aid = ArrayId::new(TenantId::new(1), "arr_col"); + let schema = array_schema_1d("arr_col"); + let schema_bytes = zerompk::to_msgpack_vec(&schema).unwrap(); + + send_ok( + core, + tx, + rx, + PhysicalPlan::Array(ArrayOp::OpenArray { + array_id: aid.clone(), + schema_msgpack: schema_bytes, + schema_hash: 1, + prefix_bits: 8, + audit_retain_ms: None, + minimum_audit_retain_ms: None, + }), + ); + + let cells: Vec = ARR_SURS + .iter() + .copied() + .enumerate() + .map(|(i, s)| ArrayPutCell { + coord: vec![CoordValue::Int64(i as i64)], + attrs: vec![CellValue::Float64(s as f64)], + surrogate: Surrogate::new(s), + system_from_ms: 0, + valid_from_ms: 0, + valid_until_ms: i64::MAX, + }) + .collect(); + let cells_mp = zerompk::to_msgpack_vec(&cells).unwrap(); + + send_ok( + core, + tx, + rx, + PhysicalPlan::Array(ArrayOp::Put { + array_id: aid.clone(), + cells_msgpack: cells_mp, + wal_lsn: 1, + provenance: None, + vshard_id: 0, + }), + ); +} + +// ── 7. FTS inserts (via Document engine; text is indexed separately) ─ +pub(super) fn insert_fts( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, +) { + for &s in FTS_SURS { + let hex = format!("{s:08x}"); + let doc = serde_json::json!({ "id": hex, "body": format!("learning keyword {s}") }); + send_ok( + core, + tx, + rx, + PhysicalPlan::Document(DocumentOp::PointPut { + collection: nodedb_types::QualifiedCollection::new( + nodedb_types::DatabaseId::DEFAULT, + "fts_col", + ), + document_id: hex.clone(), + value: serde_json::to_vec(&doc).unwrap(), + surrogate: Surrogate::new(s), + pk_bytes: hex.into_bytes(), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + }), + ); + } +} + +// ── 8. Spatial inserts (via Document engine; geometry stored as field) ─ +pub(super) fn insert_spatial( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, +) { + for &s in SPATIAL_SURS { + let hex = format!("{s:08x}"); + let lon = s as f64 * 0.01; + let doc = serde_json::json!({ "id": hex, "geom": { "type": "Point", "coordinates": [lon, 0.0] } }); + send_ok( + core, + tx, + rx, + PhysicalPlan::Document(DocumentOp::PointPut { + collection: nodedb_types::QualifiedCollection::new( + nodedb_types::DatabaseId::DEFAULT, + "spatial_col", + ), + document_id: hex.clone(), + value: serde_json::to_vec(&doc).unwrap(), + surrogate: Surrogate::new(s), + pk_bytes: hex.into_bytes(), + returning: None, + rls_filters: Vec::new(), + resolved_sum_targets: Vec::new(), + }), + ); + } +} diff --git a/nodedb/tests/inproc/cases/surrogate_round_trip/intersection.rs b/nodedb/tests/inproc/cases/surrogate_round_trip/intersection.rs new file mode 100644 index 000000000..1b3cbe712 --- /dev/null +++ b/nodedb/tests/inproc/cases/surrogate_round_trip/intersection.rs @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: BUSL-1.1 + +use nodedb::bridge::dispatch::{BridgeRequest, BridgeResponse}; +use nodedb::data::executor::core_loop::CoreLoop; +use nodedb::data::executor::response_codec::decode_payload_to_json; +use nodedb::engine::graph::edge_store::Direction; +use nodedb::types::TenantId; +use nodedb_array::query::slice::Slice as ArraySlice; +use nodedb_array::types::ArrayId; +use nodedb_bridge::buffer::{Consumer, Producer}; +use nodedb_physical::physical_plan::{ArrayOp, GraphOp, PhysicalPlan, VectorOp}; +use nodedb_types::vector_distance::DistanceMetric; +use nodedb_types::{Surrogate, SurrogateBitmap}; + +use super::harness::{ARR_SURS, G_SRC_SURS, V_SURS, extract_vector_surrogates, send_ok}; + +// ══════════════════════════════════════════════════════════════════════ +// Assertion B: Cross-engine bitmap intersection (Vector ∩ Graph ∩ Array). +// +// Design: +// bitmap_v = {10, 11, 12, 13} (all Vector surrogates) +// bitmap_g = {10, 12, 30, 31} (all Graph src surrogates) +// bitmap_a = {10, 13, 60, 61} (all Array cell surrogates) +// intersection = bitmap_v ∩ bitmap_g ∩ bitmap_a = {10} +// +// For each engine a query runs with `intersection` as the prefilter +// and the assertions check that only surrogate 10 is returned. +// ══════════════════════════════════════════════════════════════════════ +pub(super) fn assert_bitmap_intersection( + core: &mut CoreLoop, + tx: &mut Producer, + rx: &mut Consumer, +) { + let bitmap_v: SurrogateBitmap = V_SURS.iter().copied().map(Surrogate::new).collect(); + let bitmap_g: SurrogateBitmap = G_SRC_SURS.iter().copied().map(Surrogate::new).collect(); + let bitmap_a: SurrogateBitmap = ARR_SURS.iter().copied().map(Surrogate::new).collect(); + + let intersection = bitmap_v.intersect(&bitmap_g).intersect(&bitmap_a); + + assert_eq!( + intersection.len(), + 1, + "three-way intersection must contain exactly surrogate 10" + ); + assert!( + intersection.contains(Surrogate::new(10)), + "surrogate 10 must be in the intersection" + ); + + // B1 — Vector search with intersection as filter_bitmap. + // Unfiltered search returns all four Vector surrogates (11, 12, 13 + // also match the query vector near [1,0,0]). + let all_vec = send_ok( + core, + tx, + rx, + PhysicalPlan::Vector(VectorOp::Search { + collection: nodedb_types::QualifiedCollection::new( + nodedb_types::DatabaseId::DEFAULT, + "vec_col", + ), + query_vector: vec![1.0f32, 0.0, 0.0], + top_k: 20, + ef_search: 0, + filter_bitmap: None, + field_name: String::new(), + rls_filters: Vec::new(), + inline_prefilter_plan: None, + ann_options: Default::default(), + skip_payload_fetch: false, + payload_filters: Vec::new(), + metric: DistanceMetric::L2, + }), + ); + let all_vec_surs = extract_vector_surrogates(&all_vec); + assert!( + all_vec_surs.len() >= 2, + "unfiltered vector search should return at least 2 results, got {all_vec_surs:?}" + ); + + let filtered_vec = send_ok( + core, + tx, + rx, + PhysicalPlan::Vector(VectorOp::Search { + collection: nodedb_types::QualifiedCollection::new( + nodedb_types::DatabaseId::DEFAULT, + "vec_col", + ), + query_vector: vec![1.0f32, 0.0, 0.0], + top_k: 20, + ef_search: 0, + filter_bitmap: Some(intersection.clone()), + field_name: String::new(), + rls_filters: Vec::new(), + inline_prefilter_plan: None, + ann_options: Default::default(), + skip_payload_fetch: false, + payload_filters: Vec::new(), + metric: DistanceMetric::L2, + }), + ); + let filtered_vec_surs = extract_vector_surrogates(&filtered_vec); + + // Every returned surrogate must be in the intersection. + for &s in &filtered_vec_surs { + assert!( + intersection.contains(Surrogate::new(s)), + "vector search returned surrogate {s} not in intersection" + ); + } + // Surrogates 11, 12, 13 must not appear (not in intersection). + for &absent in &[11u32, 12, 13] { + assert!( + !filtered_vec_surs.contains(&absent), + "surrogate {absent} is Vector-only or Vector+one-engine; must be absent from intersection-filtered search" + ); + } + + // B2 — Graph hop with frontier_bitmap = intersection. + // An inserted edge n10 → n11 makes n11 reachable. + // With frontier_bitmap = {10}, the hop does not traverse to n11 + // (surrogate 11 is not in the intersection). + send_ok( + core, + tx, + rx, + PhysicalPlan::Graph(GraphOp::EdgePut { + collection: nodedb_types::QualifiedCollection::new( + nodedb_types::DatabaseId::DEFAULT, + "graph_col", + ), + src_id: "n10".into(), + label: "REACH".into(), + dst_id: "n11".into(), + properties: vec![], + src_surrogate: Surrogate::new(10), + dst_surrogate: Surrogate::new(11), + }), + ); + + let hop_all = send_ok( + core, + tx, + rx, + PhysicalPlan::Graph(GraphOp::Hop { + start_nodes: vec!["n10".into()], + edge_labels: vec!["REACH".into()], + direction: Direction::Out, + depth: 1, + options: Default::default(), + rls_filters: Vec::new(), + frontier_bitmap: None, + collection: None, + }), + ); + let hop_all_json = decode_payload_to_json(&hop_all); + // Unfiltered: n11 is reachable from n10 via REACH. + assert!( + hop_all_json.contains("n11"), + "unfiltered hop from n10 must reach n11: {hop_all_json}" + ); + + let hop_filtered = send_ok( + core, + tx, + rx, + PhysicalPlan::Graph(GraphOp::Hop { + start_nodes: vec!["n10".into()], + edge_labels: vec!["REACH".into()], + direction: Direction::Out, + depth: 1, + options: Default::default(), + rls_filters: Vec::new(), + frontier_bitmap: Some(intersection.clone()), + collection: None, + }), + ); + let hop_filtered_json = decode_payload_to_json(&hop_filtered); + // Filtered: n11's surrogate (11) is not in the intersection — the hop + // must not include it. + assert!( + !hop_filtered_json.contains("n11"), + "intersection-filtered hop must not reach n11 (surrogate 11 not in intersection): \ + {hop_filtered_json}" + ); + + // B3 — Array slice with cell_filter = intersection. + { + let aid = ArrayId::new(TenantId::new(1), "arr_col"); + // Full slice (no filter). + // Full slice: no restrictions on any dimension (None = unbounded). + let full_slice = ArraySlice::new(vec![None]); + let slice_mp = zerompk::to_msgpack_vec(&full_slice).unwrap(); + + let slice_all = send_ok( + core, + tx, + rx, + PhysicalPlan::Array(ArrayOp::Slice { + array_id: aid.clone(), + slice_msgpack: slice_mp.clone(), + attr_projection: vec![], + limit: 100, + cell_filter: None, + shard_bucket: None, + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + zone_msgpack: Vec::new(), + }), + ); + // Unfiltered: returns all ARR_SURS cells. + let slice_all_json = decode_payload_to_json(&slice_all); + assert!( + !slice_all_json.is_empty(), + "unfiltered array slice must return rows" + ); + + let slice_filtered = send_ok( + core, + tx, + rx, + PhysicalPlan::Array(ArrayOp::Slice { + array_id: aid.clone(), + slice_msgpack: slice_mp, + attr_projection: vec![], + limit: 100, + cell_filter: Some(intersection.clone()), + shard_bucket: None, + system_time: nodedb_types::SystemTimeScope::Current, + valid_at_ms: None, + zone_msgpack: Vec::new(), + }), + ); + // Filtered: intersection ∩ ARR_SURS = {10}. Surrogate 10 is at + // coord [0] with attr value 10.0 (see the array insert in `inserts`). + // Surrogates 13 (coord [1]), 60 (coord [2]), 61 (coord [3]) must not + // appear. The response is zerompk-encoded ArrayCell values; they + // are transcoded to JSON for inspection. + // Local DP slice responses are wrapped as `ArraySliceResponse { rows_msgpack: + // Vec, truncated_before_horizon: bool }`. The inner `rows_msgpack` is a + // separate msgpack array of `ArrayCell` values. + #[derive(serde::Deserialize, zerompk::FromMessagePack)] + #[msgpack(map)] + struct SliceEnvelope { + rows_msgpack: Vec, + truncated_before_horizon: bool, + } + fn unwrap_rows(payload: &[u8]) -> serde_json::Value { + let envelope: SliceEnvelope = + zerompk::from_msgpack(payload).expect("slice response envelope decodes"); + assert!( + !envelope.truncated_before_horizon, + "a current slice is never truncated before the audit horizon" + ); + let json_str = nodedb_types::msgpack_to_json_string(&envelope.rows_msgpack) + .expect("slice rows decode"); + serde_json::from_str(&json_str).expect("slice rows are JSON") + } + let filtered_cells = unwrap_rows(&slice_filtered); + let unfiltered_cells = unwrap_rows(&slice_all); + assert_eq!( + unfiltered_cells.as_array().map(|a| a.len()).unwrap_or(0), + ARR_SURS.len(), + "unfiltered slice must return all {} array cells", + ARR_SURS.len() + ); + + let filtered_count = filtered_cells.as_array().map(|a| a.len()).unwrap_or(0); + assert_eq!( + filtered_count, 1, + "intersection-filtered slice must return exactly 1 cell (surrogate 10), \ + got {filtered_count}: {filtered_cells}" + ); + + // The surviving cell must have attr value 10.0 (the value stored at + // coord [0], which received surrogate 10). A slice row names each attr. + let cell = filtered_cells + .as_array() + .and_then(|a| a.first()) + .expect("one filtered cell"); + assert_eq!( + cell.get("v").and_then(|v| v.as_f64()), + Some(10.0), + "filtered cell attr `v` must be 10.0 (surrogate-10's stored value): {cell}" + ); + } +} diff --git a/nodedb/tests/inproc/cases/surrogate_round_trip/mod.rs b/nodedb/tests/inproc/cases/surrogate_round_trip/mod.rs new file mode 100644 index 000000000..d88c55373 --- /dev/null +++ b/nodedb/tests/inproc/cases/surrogate_round_trip/mod.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Single-node integration test for surrogate identity round-trip. +//! +//! Asserts three properties: +//! +//! 1. **Per-engine uniqueness** — the surrogate values assigned to rows in +//! each engine are distinct from one another within the engine's own +//! insert batch (no engine re-uses a surrogate for two different rows). +//! +//! 2. **Global uniqueness** — all surrogates across every engine form a +//! set with no duplicates. +//! +//! 3. **Cross-engine bitmap intersection** — a three-way intersection of +//! `SurrogateBitmap`s produced from Vector, Graph, and Array engine +//! inserts yields the correct set, and actually running queries with +//! the intersected bitmap as `filter_bitmap` / `frontier_bitmap` / +//! `cell_filter` returns only rows whose surrogate is in the +//! intersection. +//! +//! The test drives a single `CoreLoop` via the ring-buffer harness (the +//! same pattern as `cross_engine_bitmap_currency.rs`) so no server or +//! network is required. + +mod harness; +mod inserts; +mod intersection; +mod round_trip; +mod uniqueness; diff --git a/nodedb/tests/inproc/cases/surrogate_round_trip/round_trip.rs b/nodedb/tests/inproc/cases/surrogate_round_trip/round_trip.rs new file mode 100644 index 000000000..fe04a89af --- /dev/null +++ b/nodedb/tests/inproc/cases/surrogate_round_trip/round_trip.rs @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: BUSL-1.1 + +use super::harness::open_core; +use super::inserts::{ + insert_array, insert_columnar, insert_document, insert_fts, insert_graph, insert_kv, + insert_spatial, insert_vector, +}; +use super::intersection::assert_bitmap_intersection; +use super::uniqueness::assert_surrogate_uniqueness; + +// ── Test ─────────────────────────────────────────────────────────────────── + +/// Main assertion: insert into eight engine paths, check uniqueness, check +/// the three-way Vector ∩ Graph ∩ Array bitmap intersection via real query +/// execution. +#[test] +fn surrogate_round_trip_all_engines() { + let (mut core, mut tx, mut rx, _dir) = open_core(); + + insert_vector(&mut core, &mut tx, &mut rx); + insert_document(&mut core, &mut tx, &mut rx); + insert_graph(&mut core, &mut tx, &mut rx); + insert_kv(&mut core, &mut tx, &mut rx); + insert_columnar(&mut core, &mut tx, &mut rx); + insert_array(&mut core, &mut tx, &mut rx); + insert_fts(&mut core, &mut tx, &mut rx); + insert_spatial(&mut core, &mut tx, &mut rx); + + assert_surrogate_uniqueness(); + assert_bitmap_intersection(&mut core, &mut tx, &mut rx); +} diff --git a/nodedb/tests/inproc/cases/surrogate_round_trip/uniqueness.rs b/nodedb/tests/inproc/cases/surrogate_round_trip/uniqueness.rs new file mode 100644 index 000000000..597e887b6 --- /dev/null +++ b/nodedb/tests/inproc/cases/surrogate_round_trip/uniqueness.rs @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: BUSL-1.1 + +use std::collections::HashSet; + +use super::harness::{ + ARR_SURS, COL_SURS, D_SURS, FTS_SURS, G_SRC_SURS, KV_SURS, SPATIAL_SURS, V_SURS, +}; + +// ══════════════════════════════════════════════════════════════════════ +// Assertion A: Global surrogate uniqueness across all engines. +// +// All surrogate values are controlled here; collect every value and assert +// the multi-set has no duplicates. The non-overlap is designed into the +// constants in the harness; this assertion catches any later constant change +// that accidentally introduces a collision. +// ══════════════════════════════════════════════════════════════════════ +pub(super) fn assert_surrogate_uniqueness() { + let all_surs: Vec = [ + V_SURS, + D_SURS, + G_SRC_SURS, + KV_SURS, + COL_SURS, + ARR_SURS, + FTS_SURS, + SPATIAL_SURS, + ] + .iter() + .flat_map(|s| s.iter().copied()) + .collect(); + + // Surrogates 10, 12, 13 intentionally appear in multiple engines (the + // cross-engine join scenario). The uniqueness property asserted here + // is that each engine's OWN insert batch uses distinct values — not + // that the same physical surrogate never appears in two engines (that + // would prohibit the cross-engine join design). + for slice in [ + V_SURS, + D_SURS, + G_SRC_SURS, + KV_SURS, + COL_SURS, + ARR_SURS, + FTS_SURS, + SPATIAL_SURS, + ] { + let set: HashSet = slice.iter().copied().collect(); + assert_eq!( + set.len(), + slice.len(), + "engine surrogate batch must contain no duplicates: {slice:?}" + ); + } + + // Build the global set (deduped) and assert no same surrogate maps to + // two different rows in the SAME engine. + let global_set: HashSet = all_surs.iter().copied().collect(); + // Global set has fewer elements than all_surs because of intentional + // cross-engine reuse of {10, 12, 13}. The per-engine uniqueness loop + // above asserts that every surrogate is unique within each engine batch. + assert!( + global_set.contains(&10), + "surrogate 10 must be in the global set" + ); +} diff --git a/nodedb/tests/wire/cases/array_georef.rs b/nodedb/tests/wire/cases/array_georef.rs new file mode 100644 index 000000000..e1dfa76b9 --- /dev/null +++ b/nodedb/tests/wire/cases/array_georef.rs @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Array `GEOREF`: CREATE, DESCRIBE, `ALTER ARRAY .. SET/DROP GEOREF`, and +//! the ELEMENTWISE equal-georef rule. +//! +//! A georef change keeps `schema_hash`. Two proofs run over the wire: +//! flushed segments stay readable after ALTER and after restart, and +//! ELEMENTWISE passes the Data Plane hash check after both arrays change. + +use std::collections::HashMap; + +use crate::harness::TestServer; + +const PIXEL_GT: &str = + "TRANSFORM (100.5, 8.983152841195214e-05, 0, 6.5, 0, -8.983152841195214E-05)"; + +fn create_raster(name: &str, georef: &str) -> String { + format!( + "CREATE ARRAY {name} \ + DIMS (row INT64 [0..15], col INT64 [0..15]) \ + ATTRS (b1 FLOAT64) \ + TILE_EXTENTS (4, 4) \ + TILE_ORDER ROW_MAJOR {georef}" + ) +} + +/// The SQLSTATE `sql` is refused with. +async fn sqlstate(server: &TestServer, sql: &str) -> String { + let err = server + .client + .simple_query(sql) + .await + .expect_err("statement must be refused"); + err.as_db_error() + .map(|db| db.code().code().to_string()) + .unwrap_or_else(|| panic!("no SQLSTATE on {err}")) +} + +/// DESCRIBE rows of `name`, keyed by field, valued by type. +async fn describe(server: &TestServer, name: &str) -> HashMap { + server + .query_named_rows(&format!("DESCRIBE {name}")) + .await + .unwrap_or_else(|e| panic!("DESCRIBE {name}: {e}")) + .into_iter() + .map(|row| { + let field = row.get("field").cloned().unwrap_or_default(); + let ty = row.get("type").cloned().unwrap_or_default(); + (field, ty) + }) + .collect() +} + +/// Every cell of `name` as `(row, col) -> b1`. +async fn cells(server: &TestServer, name: &str) -> HashMap<(i64, i64), f64> { + server + .query_named_rows(&format!( + "SELECT * FROM ARRAY_SLICE('{name}', '{{\"row\":[0,15],\"col\":[0,15]}}', '*', 1000)" + )) + .await + .unwrap_or_else(|e| panic!("ARRAY_SLICE {name}: {e}")) + .iter() + .map(|row| { + let num = |c: &str| row.get(c).and_then(|t| t.parse::().ok()); + let key = ( + num("row").expect("row") as i64, + num("col").expect("col") as i64, + ); + (key, num("b1").expect("b1")) + }) + .collect() +} + +#[tokio::test] +async fn create_with_georef_then_describe() { + let server = TestServer::start().await; + server + .exec(&create_raster( + "geo_pixel", + &format!("GEOREF (SRID 4326, X col, Y row, {PIXEL_GT})"), + )) + .await + .expect("CREATE with pixel GEOREF"); + let rows = describe(&server, "geo_pixel").await; + assert_eq!( + rows.get("__georef").map(String::as_str), + Some( + "SRID 4326, X col, Y row, TRANSFORM (100.5, 8.983152841195213e-5, 0.0, 6.5, 0.0, \ + -8.983152841195213e-5)" + ), + "{rows:?}" + ); + assert_eq!(rows.get("__storage").map(String::as_str), Some("array")); + assert_eq!(rows.get("__tile_extents").map(String::as_str), Some("4, 4")); + assert_eq!( + rows.get("row").map(String::as_str), + Some("INT64 DIM [0..15]") + ); + + server + .exec( + "CREATE ARRAY geo_world DIMS (lon FLOAT64 [99.0..120.0], lat FLOAT64 [0.0..8.0]) \ + ATTRS (v FLOAT64) TILE_EXTENTS (1, 1) GEOREF (SRID 4326, X lon, Y lat)", + ) + .await + .expect("CREATE with world GEOREF"); + let rows = describe(&server, "geo_world").await; + assert_eq!( + rows.get("__georef").map(String::as_str), + Some("SRID 4326, X lon, Y lat") + ); + + server + .exec(&create_raster("geo_none", "")) + .await + .expect("CREATE without GEOREF"); + assert!(!describe(&server, "geo_none").await.contains_key("__georef")); +} + +#[tokio::test] +async fn create_georef_refusals_carry_their_sqlstate() { + let server = TestServer::start().await; + let cases = [ + ("GEOREF (SRID 2154, X col, Y row)", "22000"), + ("GEOREF (SRID 4326, X col, Y lat)", "42703"), + ("GEOREF (SRID 4326, X col, Y col)", "22000"), + ( + "GEOREF (SRID 4326, X col, Y row, TRANSFORM (0, 1, 1, 0, 1, 1))", + "22000", + ), + ( + "GEOREF (SRID 4326, X col, Y row, TRANSFORM (0, 1e999, 0, 0, 0, -1))", + "22000", + ), + ( + "GEOREF (SRID 4326, X col, Y row, TRANSFORM (0, 1, 0, 0, 0))", + "22000", + ), + ]; + for (i, (clause, code)) in cases.into_iter().enumerate() { + let sql = create_raster(&format!("geo_bad_{i}"), clause); + assert_eq!(sqlstate(&server, &sql).await, code, "{clause}"); + } + let floats = "CREATE ARRAY geo_bad_float DIMS (x FLOAT64 [0.0..1.0], y FLOAT64 [0.0..1.0]) \ + ATTRS (v FLOAT64) TILE_EXTENTS (1, 1) \ + GEOREF (SRID 32647, X x, Y y, TRANSFORM (0, 10, 0, 0, 0, -10))"; + assert_eq!(sqlstate(&server, floats).await, "22000"); +} + +#[tokio::test] +async fn alter_set_and_drop_georef_keep_data_and_survive_restart() { + let server = TestServer::start().await; + server + .exec(&create_raster("geo_alter", "")) + .await + .expect("CREATE"); + let mut want = HashMap::new(); + for (row, col) in [(0, 0), (3, 7), (15, 15)] { + let v = (row * 16 + col) as f64 + 0.5; + server + .exec(&format!( + "INSERT INTO ARRAY geo_alter COORDS ({row}, {col}) VALUES ({v})" + )) + .await + .expect("INSERT before flush"); + want.insert((row, col), v); + } + server + .exec("SELECT ARRAY_FLUSH('geo_alter')") + .await + .expect("flush"); + server + .exec("INSERT INTO ARRAY geo_alter COORDS (8, 8) VALUES (136.5)") + .await + .expect("INSERT into the memtable"); + want.insert((8, 8), 136.5); + let before = describe(&server, "geo_alter").await; + + server + .exec( + "ALTER ARRAY geo_alter SET GEOREF (SRID 32647, X col, Y row, \ + TRANSFORM (600000, 10, 0, 700000, 0, -10))", + ) + .await + .expect("SET GEOREF"); + let georef = "SRID 32647, X col, Y row, TRANSFORM (600000.0, 10.0, 0.0, 700000.0, 0.0, -10.0)"; + let after = describe(&server, "geo_alter").await; + assert_eq!(after.get("__georef").map(String::as_str), Some(georef)); + let mut without = after.clone(); + without.remove("__georef"); + assert_eq!(without, before, "SET GEOREF keeps every other field"); + assert_eq!(cells(&server, "geo_alter").await, want); + + assert_eq!( + sqlstate( + &server, + "ALTER ARRAY geo_alter SET GEOREF (SRID 4326, X col, Y lat)" + ) + .await, + "42703" + ); + assert_eq!( + sqlstate( + &server, + "ALTER ARRAY geo_alter SET GEOREF (SRID 9999, X col, Y row)" + ) + .await, + "22000" + ); + + let (server, dir) = server.take_dir(); + server.graceful_shutdown().await; + let (server, dir) = TestServer::open_on_path(dir).await; + let restarted = describe(&server, "geo_alter").await; + assert_eq!(restarted, after, "the georef survives restart"); + assert_eq!(cells(&server, "geo_alter").await, want); + + server + .exec("ALTER ARRAY geo_alter DROP GEOREF") + .await + .expect("DROP GEOREF"); + assert_eq!(describe(&server, "geo_alter").await, before); + assert_eq!( + sqlstate(&server, "ALTER ARRAY geo_alter DROP GEOREF").await, + "55000" + ); + + // A server opened on a path holds a placeholder; `dir` is the real one. + server.graceful_shutdown().await; + let (server, _dir) = TestServer::open_on_path(dir).await; + assert_eq!(describe(&server, "geo_alter").await, before); + assert_eq!(cells(&server, "geo_alter").await, want); +} + +#[tokio::test] +async fn elementwise_requires_equal_georefs() { + let server = TestServer::start().await; + for name in ["geo_left", "geo_right"] { + server.exec(&create_raster(name, "")).await.expect("CREATE"); + server + .exec(&format!( + "INSERT INTO ARRAY {name} COORDS (1, 2) VALUES (4.0)" + )) + .await + .expect("INSERT"); + server + .exec(&format!("SELECT ARRAY_FLUSH('{name}')")) + .await + .expect("flush"); + } + let sql = "SELECT * FROM ARRAY_ELEMENTWISE('geo_left', 'geo_right', 'add', 'b1')"; + let rows = server + .query_rows(sql) + .await + .expect("no georef on either side"); + assert_eq!(rows.len(), 1); + + let set = |name: &str, x0: &str| { + format!( + "ALTER ARRAY {name} SET GEOREF (SRID 32647, X col, Y row, \ + TRANSFORM ({x0}, 10, 0, 700000, 0, -10))" + ) + }; + server + .exec(&set("geo_left", "600000")) + .await + .expect("SET left"); + assert_eq!(sqlstate(&server, sql).await, "22000"); + server + .exec(&set("geo_right", "600010")) + .await + .expect("SET right"); + assert_eq!(sqlstate(&server, sql).await, "22000"); + server + .exec(&set("geo_right", "600000")) + .await + .expect("SET right equal"); + // The Data Plane refuses unequal schema hashes, so success proves both + // ALTERs kept the hash. + let rows = server.query_rows(sql).await.expect("equal georefs"); + assert_eq!(rows.len(), 1); +} diff --git a/nodedb/tests/wire/cases/array_georef_zone.rs b/nodedb/tests/wire/cases/array_georef_zone.rs new file mode 100644 index 000000000..4066fe624 --- /dev/null +++ b/nodedb/tests/wire/cases/array_georef_zone.rs @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! SRID zones and cell geometry over georeferenced arrays. +//! +//! Two arrays hold the same 32x32 grid of 30 m UTM 47N pixels: `zpx` as +//! pixel indices with a TRANSFORM, `zwd` as pixel-centre world coordinates. +//! A zone drawn in WGS 84 must select the same cells from both. The +//! expected cells come from projecting each centre to WGS 84 in the test. + +use std::collections::BTreeSet; + +use nodedb_spatial::crs::Transformer; +use nodedb_types::geometry::{Srid, point_in_polygon}; + +use crate::harness::TestServer; + +const SIZE: i64 = 32; +const PIXEL_M: f64 = 30.0; +const EAST0: f64 = 550_000.0; +const NORTH0: f64 = 664_000.0; + +/// World centre of pixel `(row, col)`, UTM 47N metres. +fn centre(row: i64, col: i64) -> [f64; 2] { + [ + EAST0 + (col as f64 + 0.5) * PIXEL_M, + NORTH0 - (row as f64 + 0.5) * PIXEL_M, + ] +} + +fn cell_value(row: i64, col: i64) -> f64 { + (row * SIZE + col) as f64 +} + +/// A WGS 84 quadrilateral over the middle of the grid, from UTM corners. +fn zone_ring() -> Vec<[f64; 2]> { + let to_wgs84 = Transformer::new(Srid::Utm47N, Srid::Wgs84); + let utm = [ + [EAST0 + 140.0, NORTH0 - 110.0], + [EAST0 + 830.0, NORTH0 - 260.0], + [EAST0 + 700.0, NORTH0 - 880.0], + [EAST0 + 210.0, NORTH0 - 760.0], + ]; + let mut ring: Vec<[f64; 2]> = utm + .iter() + .map(|p| to_wgs84.apply(*p).expect("inside UTM 47N")) + .collect(); + ring.push(ring[0]); + ring +} + +/// `ring` as an EWKT literal in SRID 4326. +fn ewkt(ring: &[[f64; 2]]) -> String { + let points: Vec = ring + .iter() + .map(|p| format!("{:?} {:?}", p[0], p[1])) + .collect(); + format!("'SRID=4326;POLYGON(({}))'", points.join(", ")) +} + +/// The cells whose centre lies inside `ring`. +fn expected_cells(ring: &[[f64; 2]]) -> BTreeSet<(i64, i64)> { + let to_wgs84 = Transformer::new(Srid::Utm47N, Srid::Wgs84); + let mut out = BTreeSet::new(); + for row in 0..SIZE { + for col in 0..SIZE { + let [lon, lat] = to_wgs84.apply(centre(row, col)).expect("inside UTM 47N"); + if point_in_polygon(lon, lat, ring) { + out.insert((row, col)); + } + } + } + out +} + +/// Create `zpx` (pixel mode) and `zwd` (world mode), fill and flush both. +async fn create_grids(server: &TestServer) { + server + .exec(&format!( + "CREATE ARRAY zpx DIMS (row INT64 [0..{last}], col INT64 [0..{last}]) \ + ATTRS (b1 FLOAT64) TILE_EXTENTS (8, 8) TILE_ORDER ROW_MAJOR \ + GEOREF (SRID 32647, X col, Y row, TRANSFORM ({EAST0}, {PIXEL_M}, 0, {NORTH0}, 0, \ + -{PIXEL_M}))", + last = SIZE - 1 + )) + .await + .expect("CREATE ARRAY zpx"); + server + .exec( + "CREATE ARRAY zwd DIMS (x FLOAT64 [550000..551000], y FLOAT64 [663000..664000]) \ + ATTRS (b1 FLOAT64) TILE_EXTENTS (240, 240) TILE_ORDER ROW_MAJOR \ + GEOREF (SRID 32647, X x, Y y)", + ) + .await + .expect("CREATE ARRAY zwd"); + for row in 0..SIZE { + let pixel: Vec = (0..SIZE) + .map(|col| format!("COORDS ({row}, {col}) VALUES ({:?})", cell_value(row, col))) + .collect(); + server + .exec(&format!("INSERT INTO ARRAY zpx {}", pixel.join(", "))) + .await + .expect("INSERT zpx"); + let world: Vec = (0..SIZE) + .map(|col| { + let [x, y] = centre(row, col); + format!("COORDS ({x:?}, {y:?}) VALUES ({:?})", cell_value(row, col)) + }) + .collect(); + server + .exec(&format!("INSERT INTO ARRAY zwd {}", world.join(", "))) + .await + .expect("INSERT zwd"); + } + for name in ["zpx", "zwd"] { + server + .exec(&format!("SELECT ARRAY_FLUSH('{name}')")) + .await + .expect("ARRAY_FLUSH"); + } +} + +/// The SQLSTATE `sql` is refused with. +async fn sqlstate(server: &TestServer, sql: &str) -> String { + let err = server + .client + .simple_query(sql) + .await + .expect_err("statement must be refused"); + err.as_db_error() + .map(|db| db.code().code().to_string()) + .unwrap_or_else(|| panic!("no SQLSTATE on {err}")) +} + +fn number(row: &std::collections::HashMap, col: &str) -> f64 { + row.get(col) + .and_then(|t| t.parse::().ok()) + .unwrap_or_else(|| panic!("column {col} is not a number in {row:?}")) +} + +async fn scalar(server: &TestServer, sql: &str) -> f64 { + let rows = server + .query_named_rows(sql) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")); + assert_eq!(rows.len(), 1, "{sql}: one scalar row: {rows:?}"); + number(&rows[0], "result") +} + +/// `(row, col)` of every pixel `zpx` slice row. +async fn pixel_cells(server: &TestServer, sql: &str) -> BTreeSet<(i64, i64)> { + server + .query_named_rows(sql) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")) + .iter() + .map(|r| (number(r, "row") as i64, number(r, "col") as i64)) + .collect() +} + +#[tokio::test] +async fn agg_over_a_wgs84_zone_matches_on_pixel_and_world_arrays() { + let server = TestServer::start().await; + create_grids(&server).await; + let ring = zone_ring(); + let expected = expected_cells(&ring); + assert!( + expected.len() > 50, + "the zone holds cells: {}", + expected.len() + ); + let expected_sum: f64 = expected.iter().map(|&(r, c)| cell_value(r, c)).sum(); + let zone = ewkt(&ring); + for name in ["zpx", "zwd"] { + let count = scalar( + &server, + &format!("SELECT * FROM ARRAY_AGG('{name}', 'b1', 'count', NULL, NULL, {zone}, NULL)"), + ) + .await; + let sum = scalar( + &server, + &format!("SELECT * FROM ARRAY_AGG('{name}', 'b1', 'sum', NULL, NULL, {zone}, NULL)"), + ) + .await; + assert_eq!(count, expected.len() as f64, "{name}: count"); + assert!( + (sum - expected_sum).abs() < 1e-6, + "{name}: sum {sum} vs {expected_sum}" + ); + } +} + +#[tokio::test] +async fn slice_with_a_wgs84_zone_returns_the_cells_inside_it() { + let server = TestServer::start().await; + create_grids(&server).await; + let ring = zone_ring(); + let zone = ewkt(&ring); + let got = pixel_cells( + &server, + &format!( + "SELECT * FROM ARRAY_SLICE('zpx', '{{row: [0, 31]}}', '*', 0, {zone}, \ + ARRAY['col', 'row'])" + ), + ) + .await; + assert_eq!(got, expected_cells(&ring)); + + let world_rows = server + .query_named_rows(&format!( + "SELECT * FROM ARRAY_SLICE('zwd', '{{x: [550000, 551000]}}', '*', 0, {zone}, NULL)" + )) + .await + .expect("world slice"); + assert_eq!( + world_rows.len(), + got.len(), + "world and pixel select the same cells" + ); +} + +#[tokio::test] +async fn slice_all_versions_with_a_zone_keeps_both_versions_of_inside_cells() { + let server = TestServer::start().await; + create_grids(&server).await; + let ring = zone_ring(); + let expected = expected_cells(&ring); + let inside = *expected.iter().next().expect("an inside cell"); + let outside = (0, 0); + assert!( + !expected.contains(&outside), + "the corner pixel is outside the zone" + ); + server + .exec(&format!( + "INSERT INTO ARRAY zpx COORDS ({}, {}) VALUES (-1.0), COORDS (0, 0) VALUES (-2.0)", + inside.0, inside.1 + )) + .await + .expect("overwrite"); + server + .exec("SELECT ARRAY_FLUSH('zpx')") + .await + .expect("ARRAY_FLUSH"); + let rows = server + .query_named_rows(&format!( + "SELECT * FROM ARRAY_SLICE('zpx', '{{row: [0, 31]}}', '*', 0, {}, NULL) \ + AS OF SYSTEM TIME NULL", + ewkt(&ring) + )) + .await + .expect("audit slice"); + let versions_of = |cell: (i64, i64)| { + rows.iter() + .filter(|r| (number(r, "row") as i64, number(r, "col") as i64) == cell) + .count() + }; + assert_eq!(versions_of(inside), 2, "both versions of the inside cell"); + assert_eq!(versions_of(outside), 0, "no version of the outside cell"); + assert_eq!( + rows.len(), + expected.len() + 1, + "one extra version, nothing outside" + ); +} + +#[tokio::test] +async fn a_zone_beside_the_box_narrows_to_no_rows() { + let server = TestServer::start().await; + create_grids(&server).await; + // 100.5°E is about 166 km east of the grid, still inside UTM 47N. + let far = "'SRID=4326;POLYGON((100.5 6, 100.6 6, 100.6 6.1, 100.5 6.1, 100.5 6))'"; + let rows = server + .query_named_rows(&format!( + "SELECT * FROM ARRAY_SLICE('zpx', '{{col: [0, 31]}}', '*', 0, {far}, NULL)" + )) + .await + .expect("an empty narrowed range is not an error"); + assert!(rows.is_empty(), "{rows:?}"); + let groups = server + .query_named_rows(&format!( + "SELECT * FROM ARRAY_AGG('zpx', 'b1', 'count', 'row', '{{col: [0, 31]}}', {far}, NULL)" + )) + .await + .expect("an empty narrowed range is not an error"); + assert!(groups.is_empty(), "{groups:?}"); +} + +#[tokio::test] +async fn zone_refusals_carry_their_sqlstate() { + let server = TestServer::start().await; + create_grids(&server).await; + server + .exec( + "CREATE ARRAY zplain DIMS (row INT64 [0..3], col INT64 [0..3]) ATTRS (b1 FLOAT64) \ + TILE_EXTENTS (2, 2)", + ) + .await + .expect("CREATE ARRAY zplain"); + let zone = ewkt(&zone_ring()); + let no_georef = + format!("SELECT * FROM ARRAY_AGG('zplain', 'b1', 'count', NULL, NULL, {zone}, NULL)"); + assert_eq!(sqlstate(&server, &no_georef).await, "55000"); + let other_axes = format!( + "SELECT * FROM ARRAY_AGG('zpx', 'b1', 'count', NULL, NULL, {zone}, ARRAY['row', 'col'])" + ); + assert_eq!(sqlstate(&server, &other_axes).await, "55000"); + let out_of_domain = "SELECT * FROM ARRAY_AGG('zpx', 'b1', 'count', NULL, NULL, \ + 'SRID=4326;POLYGON((150 6, 151 6, 151 7, 150 6))', NULL)"; + assert_eq!(sqlstate(&server, out_of_domain).await, "22000"); + + server + .exec(&format!( + "CREATE ARRAY ztiny DIMS (row INT64 [0..1000], col INT64 [0..1000]) \ + ATTRS (b1 FLOAT64) TILE_EXTENTS (10, 10) \ + GEOREF (SRID 32647, X col, Y row, TRANSFORM ({EAST0}, 1e-9, 0, {NORTH0}, 0, -1e-9))" + )) + .await + .expect("CREATE ARRAY ztiny"); + let long_edges = "SELECT * FROM ARRAY_AGG('ztiny', 'b1', 'count', NULL, NULL, \ + 'SRID=4326;POLYGON((99 6, 100 6, 101 6, 102 6, 102 6.5, 101 6.5, 100 6.5, 99 6.5, \ + 99 6))', NULL)"; + assert_eq!(sqlstate(&server, long_edges).await, "54000"); +} + +#[tokio::test] +async fn cell_point_transforms_like_a_hand_built_point() { + let server = TestServer::start().await; + create_grids(&server).await; + let rows = server + .query_named_rows( + "SELECT row, col, ST_X(ST_Transform(ARRAY_CELL_POINT(), 4326)) AS lon, \ + ST_Y(ST_Transform(ARRAY_CELL_POINT(), 4326)) AS lat \ + FROM ARRAY_SLICE('zpx', '{row: [0, 2], col: [0, 2]}')", + ) + .await + .expect("cell point"); + assert_eq!(rows.len(), 9, "{rows:?}"); + for row in &rows { + let (r, c) = (number(row, "row") as i64, number(row, "col") as i64); + let [x, y] = centre(r, c); + let hand = server + .query_named_rows(&format!( + "SELECT ST_X(ST_Transform(ST_SetSRID(ST_Point({x:e}, {y:e}), 32647), 4326)) \ + AS lon, ST_Y(ST_Transform(ST_SetSRID(ST_Point({x:e}, {y:e}), 32647), 4326)) \ + AS lat" + )) + .await + .expect("hand-built point"); + for axis in ["lon", "lat"] { + let (cell, built) = (number(row, axis), number(&hand[0], axis)); + assert!( + (cell - built).abs() <= 1e-12, + "({r}, {c}) {axis}: {cell} vs {built}" + ); + } + } + + let world = server + .query_named_rows( + "SELECT x, ST_X(ARRAY_CELL_POINT()) AS px FROM ARRAY_SLICE('zwd', \ + '{x: [550000, 550100], y: [663900, 664000]}')", + ) + .await + .expect("world cell point"); + assert!(!world.is_empty()); + for row in &world { + assert_eq!( + number(row, "x"), + number(row, "px"), + "world mode reads the dims" + ); + } +} + +#[tokio::test] +async fn cell_footprint_refusals_and_shape() { + let server = TestServer::start().await; + create_grids(&server).await; + server + .exec( + "CREATE ARRAY zbare DIMS (row INT64 [0..3], col INT64 [0..3]) ATTRS (b1 FLOAT64) \ + TILE_EXTENTS (2, 2)", + ) + .await + .expect("CREATE ARRAY zbare"); + let world = "SELECT ARRAY_CELL_FOOTPRINT() FROM ARRAY_SLICE('zwd', '{x: [550000, 551000]}')"; + assert_eq!(sqlstate(&server, world).await, "0A000"); + for call in ["ARRAY_CELL_FOOTPRINT()", "ARRAY_CELL_POINT()"] { + let bare = format!("SELECT {call} FROM ARRAY_SLICE('zbare', '{{row: [0, 3]}}')"); + assert_eq!(sqlstate(&server, &bare).await, "55000", "{call}"); + } + let rows = server + .query_named_rows( + "SELECT ST_NPoints(ARRAY_CELL_FOOTPRINT()) AS n, ST_Area(ARRAY_CELL_FOOTPRINT()) AS a \ + FROM ARRAY_SLICE('zpx', '{row: [0, 0], col: [0, 1]}')", + ) + .await + .expect("footprint"); + assert_eq!(rows.len(), 2, "{rows:?}"); + for row in &rows { + assert_eq!(number(row, "n"), 5.0, "a closed four-corner ring"); + assert!( + (number(row, "a") - PIXEL_M * PIXEL_M).abs() < 1e-6, + "{row:?}" + ); + } +} diff --git a/nodedb/tests/wire/cases/array_geotiff/fixture.rs b/nodedb/tests/wire/cases/array_geotiff/fixture.rs new file mode 100644 index 000000000..ceb1480b4 --- /dev/null +++ b/nodedb/tests/wire/cases/array_geotiff/fixture.rs @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The scene GeoTIFF, written in the test with the `tiff` encoder. +//! +//! 256 x 192 pixels, 3 bands of u16, chunky 64 x 64 tiles, deflate with +//! horizontal predictor 2. The geotransform is +//! `[600000, 30, 0, 700000, 0, -30]` in EPSG:32647, written as a tiepoint, +//! a pixel scale and GeoKeys. `GDAL_NODATA` is `0`, and one 10 x 10 block +//! holds 0 in every band. Every other sample is +//! `b_k(r, c) = 1000 k + (7 r + 13 c) mod 997`. + +use std::io::{Cursor, Write}; +use std::ops::Range; + +use tiff::encoder::TiffEncoder; +use tiff::encoder::compression::{CompressionAlgorithm, Deflate}; +use tiff::tags::Tag; + +pub const WIDTH: u32 = 256; +pub const HEIGHT: u32 = 192; +pub const TILE: u32 = 64; +pub const BANDS: u32 = 3; +/// The file geotransform, pixel-corner, GDAL order. +pub const GT: [f64; 6] = [600_000.0, 30.0, 0.0, 700_000.0, 0.0, -30.0]; +/// The all-zero block: rows 100..110, cols 120..130. +pub const NODATA_ROWS: Range = 100..110; +pub const NODATA_COLS: Range = 120..130; +/// Pixels the file holds minus the all-nodata block. +pub const KEPT_PIXELS: u64 = (WIDTH * HEIGHT) as u64 - 100; + +/// GeoKey ids and values. +const GT_MODEL_TYPE: u16 = 1024; +const GT_RASTER_TYPE: u16 = 1025; +const PROJECTED_CS_TYPE: u16 = 3072; +const MODEL_PROJECTED: u16 = 1; +const RASTER_PIXEL_IS_AREA: u16 = 1; +const EPSG_UTM_47N: u16 = 32647; + +/// Whether pixel `(r, c)` lies in the all-zero block. +pub fn in_nodata_block(r: u32, c: u32) -> bool { + NODATA_ROWS.contains(&r) && NODATA_COLS.contains(&c) +} + +/// The value of band `k` (1-based) at pixel `(r, c)`. +pub fn sample(k: u32, r: u32, c: u32) -> u16 { + if in_nodata_block(r, c) { + return 0; + } + let value = 1000 * k + (7 * r + 13 * c) % 997; + u16::try_from(value).expect("band values fit u16") +} + +/// The world position of the centre of pixel `(r, c)`. +pub fn pixel_centre(r: u32, c: u32) -> (f64, f64) { + let (u, v) = (f64::from(c) + 0.5, f64::from(r) + 0.5); + (GT[0] + u * GT[1] + v * GT[2], GT[3] + u * GT[4] + v * GT[5]) +} + +/// The encoded bytes of tile `(tx, ty)`: chunky samples, horizontal +/// differencing per row at the pixel stride, then deflate. +fn tile_bytes(tx: u32, ty: u32) -> Vec { + let mut raw = Vec::with_capacity((TILE * TILE * BANDS * 2) as usize); + let mut row_samples = Vec::with_capacity((TILE * BANDS) as usize); + for r in 0..TILE { + row_samples.clear(); + let row = ty * TILE + r; + for c in 0..TILE { + let col = tx * TILE + c; + row_samples.extend((1..=BANDS).map(|k| sample(k, row, col))); + } + let stride = BANDS as usize; + for (i, value) in row_samples.iter().enumerate() { + let encoded = if i < stride { + *value + } else { + value.wrapping_sub(row_samples[i - stride]) + }; + raw.extend_from_slice(&encoded.to_ne_bytes()); + } + } + let mut out = Vec::new(); + Deflate::default() + .write_to(&mut out, &raw) + .expect("deflate writes"); + out +} + +/// The whole scene GeoTIFF. +pub fn scene_tiff() -> Vec { + let mut cursor = Cursor::new(Vec::new()); + { + let mut encoder = TiffEncoder::new(&mut cursor).expect("encoder"); + let mut dir = encoder.image_directory().expect("directory"); + let mut offsets = Vec::new(); + let mut counts = Vec::new(); + for ty in 0..HEIGHT / TILE { + for tx in 0..WIDTH / TILE { + let tile = tile_bytes(tx, ty); + let offset = dir.write_data(&tile[..]).expect("tile writes"); + offsets.push(u32::try_from(offset).expect("small file")); + counts.push(u32::try_from(tile.len()).expect("small tile")); + } + } + let bands = BANDS as usize; + dir.write_tag(Tag::ImageWidth, WIDTH).expect("tag"); + dir.write_tag(Tag::ImageLength, HEIGHT).expect("tag"); + dir.write_tag(Tag::BitsPerSample, &vec![16u16; bands][..]) + .expect("tag"); + dir.write_tag(Tag::SampleFormat, &vec![1u16; bands][..]) + .expect("tag"); + dir.write_tag(Tag::SamplesPerPixel, BANDS as u16) + .expect("tag"); + dir.write_tag(Tag::PhotometricInterpretation, 1u16) + .expect("tag"); + dir.write_tag(Tag::PlanarConfiguration, 1u16).expect("tag"); + dir.write_tag(Tag::Compression, 8u16).expect("tag"); + dir.write_tag(Tag::Predictor, 2u16).expect("tag"); + dir.write_tag(Tag::TileWidth, TILE).expect("tag"); + dir.write_tag(Tag::TileLength, TILE).expect("tag"); + dir.write_tag(Tag::TileOffsets, &offsets[..]).expect("tag"); + dir.write_tag(Tag::TileByteCounts, &counts[..]) + .expect("tag"); + dir.write_tag(Tag::ModelPixelScaleTag, &[GT[1], -GT[5], 0.0][..]) + .expect("tag"); + dir.write_tag( + Tag::ModelTiepointTag, + &[0.0, 0.0, 0.0, GT[0], GT[3], 0.0][..], + ) + .expect("tag"); + let geokeys: [u16; 16] = [ + 1, + 1, + 0, + 3, + GT_MODEL_TYPE, + 0, + 1, + MODEL_PROJECTED, + GT_RASTER_TYPE, + 0, + 1, + RASTER_PIXEL_IS_AREA, + PROJECTED_CS_TYPE, + 0, + 1, + EPSG_UTM_47N, + ]; + dir.write_tag(Tag::GeoKeyDirectoryTag, &geokeys[..]) + .expect("tag"); + dir.write_tag(Tag::GdalNodata, "0").expect("tag"); + dir.finish().expect("directory writes"); + } + cursor.into_inner() +} + +/// `bytes` in a temp file whose path ends in `.tif`. The file lives as +/// long as the handle. +pub fn write_tiff(bytes: &[u8]) -> tempfile::NamedTempFile { + let mut file = tempfile::Builder::new() + .suffix(".tif") + .tempfile() + .expect("temp file"); + file.write_all(bytes).expect("temp file writes"); + file.flush().expect("temp file flushes"); + file +} + +/// The scene in a temp file, and its absolute path. +pub fn scene_file() -> (tempfile::NamedTempFile, String) { + let file = write_tiff(&scene_tiff()); + let path = file.path().to_string_lossy().into_owned(); + (file, path) +} diff --git a/nodedb/tests/wire/cases/array_geotiff/mod.rs b/nodedb/tests/wire/cases/array_geotiff/mod.rs new file mode 100644 index 000000000..3db86a0bf --- /dev/null +++ b/nodedb/tests/wire/cases/array_geotiff/mod.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: BUSL-1.1 + +pub mod fixture; +pub mod support; +pub mod utm; diff --git a/nodedb/tests/wire/cases/array_geotiff/support.rs b/nodedb/tests/wire/cases/array_geotiff/support.rs new file mode 100644 index 000000000..0cd1d9c4a --- /dev/null +++ b/nodedb/tests/wire/cases/array_geotiff/support.rs @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Arrays, COPY runs and reads shared by the GeoTIFF COPY cases. + +use std::collections::HashMap; + +use tokio_postgres::SimpleQueryMessage; + +use crate::harness::TestServer; + +/// The pixel-mode GEOREF of the scene grid. +pub const PIXEL_GEOREF: &str = + "GEOREF (SRID 32647, X col, Y row, TRANSFORM (600000, 30, 0, 700000, 0, -30))"; + +/// The three scene bands as UINT16 attrs. +pub const SCENE_ATTRS: &str = "b1 UINT16, b2 UINT16, b3 UINT16"; + +/// A pixel-mode array over the scene: INT64 `row` and `col`. +pub fn pixel_array(name: &str, rows: &str, attrs: &str, georef: &str) -> String { + format!( + "CREATE ARRAY {name} DIMS (row INT64 [{rows}], col INT64 [0..255]) \ + ATTRS ({attrs}) TILE_EXTENTS (64, 64) {georef}" + ) +} + +/// A world-mode array over the scene: FLOAT64 `y` and `x` in EPSG:32647. +pub fn world_array(name: &str) -> String { + format!( + "CREATE ARRAY {name} \ + DIMS (y FLOAT64 [694240.0..700000.0], x FLOAT64 [600000.0..607680.0]) \ + ATTRS ({SCENE_ATTRS}) TILE_EXTENTS (1920, 1920) GEOREF (SRID 32647, X x, Y y)" + ) +} + +pub async fn create(server: &TestServer, sql: &str) { + server + .exec(sql) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")); +} + +/// A refused statement: its SQLSTATE and message. +#[derive(Debug)] +pub struct Refusal { + pub code: String, + pub message: String, +} + +/// Run `COPY FROM '' WITH (FORMAT geotiff)` and +/// return the row count of its `COPY ` tag. +pub async fn copy_geotiff( + server: &TestServer, + array: &str, + path: &str, + extra: &str, +) -> Result { + let sql = format!("COPY {array} FROM '{path}' WITH (FORMAT geotiff{extra})"); + let messages = server + .client + .simple_query(&sql) + .await + .map_err(|e| match e.as_db_error() { + Some(db) => Refusal { + code: db.code().code().to_string(), + message: db.message().to_string(), + }, + None => Refusal { + code: String::new(), + message: e.to_string(), + }, + })?; + let count = messages.iter().find_map(|message| match message { + SimpleQueryMessage::CommandComplete(rows) => Some(*rows), + _ => None, + }); + Ok(count.unwrap_or_else(|| panic!("{sql}: no command tag"))) +} + +/// Run a COPY that must be refused with `code` and commit no row. +pub async fn assert_refused(server: &TestServer, array: &str, path: &str, extra: &str, code: &str) { + let err = copy_geotiff(server, array, path, extra) + .await + .expect_err("COPY must be refused"); + assert_eq!(err.code, code, "{array}: {err:?}"); + assert!( + err.message.ends_with("(0 rows committed)"), + "{array}: {err:?}" + ); + assert_eq!( + aggregate(server, array, "b1", "count", "").await, + 0.0, + "{array} holds no cell" + ); +} + +/// `ARRAY_AGG(array, attr, reducer)` as a number. No row or a NULL +/// result is 0. +pub async fn aggregate( + server: &TestServer, + array: &str, + attr: &str, + reducer: &str, + tail: &str, +) -> f64 { + let sql = format!("SELECT * FROM ARRAY_AGG('{array}', '{attr}', '{reducer}'{tail})"); + let rows = server + .query_named_rows(&sql) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")); + let Some(text) = rows.first().and_then(|row| row.get("result")) else { + return 0.0; + }; + if text.is_empty() { + return 0.0; + } + text.parse() + .unwrap_or_else(|_| panic!("{sql}: result '{text}' is not a number")) +} + +/// The cells of pixel-mode `array` in a row and col window, keyed by +/// `(row, col)`, with their `b1` text. +pub async fn pixel_cells( + server: &TestServer, + array: &str, + rows: (u32, u32), + cols: (u32, u32), +) -> HashMap<(u32, u32), String> { + let sql = format!( + "SELECT * FROM ARRAY_SLICE('{array}', '{{\"row\":[{},{}],\"col\":[{},{}]}}', '*', 100000)", + rows.0, rows.1, cols.0, cols.1 + ); + let found = server + .query_named_rows(&sql) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")); + found + .iter() + .map(|row| { + let index = |column: &str| -> u32 { + row.get(column) + .and_then(|text| text.parse::().ok()) + .unwrap_or_else(|| panic!("{column} in {row:?}")) as u32 + }; + let b1 = row.get("b1").cloned().unwrap_or_default(); + ((index("row"), index("col")), b1) + }) + .collect() +} diff --git a/nodedb/tests/wire/cases/array_geotiff/utm.rs b/nodedb/tests/wire/cases/array_geotiff/utm.rs new file mode 100644 index 000000000..114dd0ffc --- /dev/null +++ b/nodedb/tests/wire/cases/array_geotiff/utm.rs @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! An independent answer for zone queries: a UTM inverse from Snyder, +//! "Map Projections: A Working Manual" (USGS PP 1395, 1987), eqs. 8-18 to +//! 8-25, an even-odd point-in-polygon test, and a point-to-edge distance. +//! None of it shares code with the engine. + +/// WGS 84 semi-major axis, metres. +const A: f64 = 6_378_137.0; +/// WGS 84 flattening. +const F: f64 = 1.0 / 298.257_223_563; +/// UTM scale factor on the central meridian. +const K0: f64 = 0.9996; +/// UTM false easting, metres. +const FALSE_EASTING: f64 = 500_000.0; +/// Central meridian of UTM zone 47, degrees. +const ZONE_47_MERIDIAN: f64 = 99.0; + +/// `(lon, lat)` in degrees of the EPSG:32647 point `(easting, northing)`. +pub fn utm47n_to_lonlat(easting: f64, northing: f64) -> (f64, f64) { + let e2 = F * (2.0 - F); + let ep2 = e2 / (1.0 - e2); + let m = northing / K0; + let mu = m / (A * (1.0 - e2 / 4.0 - 3.0 * e2.powi(2) / 64.0 - 5.0 * e2.powi(3) / 256.0)); + let e1 = (1.0 - (1.0 - e2).sqrt()) / (1.0 + (1.0 - e2).sqrt()); + let phi1 = mu + + (3.0 * e1 / 2.0 - 27.0 * e1.powi(3) / 32.0) * (2.0 * mu).sin() + + (21.0 * e1.powi(2) / 16.0 - 55.0 * e1.powi(4) / 32.0) * (4.0 * mu).sin() + + (151.0 * e1.powi(3) / 96.0) * (6.0 * mu).sin() + + (1097.0 * e1.powi(4) / 512.0) * (8.0 * mu).sin(); + let (sin1, cos1, tan1) = (phi1.sin(), phi1.cos(), phi1.tan()); + let c1 = ep2 * cos1.powi(2); + let t1 = tan1.powi(2); + let n1 = A / (1.0 - e2 * sin1.powi(2)).sqrt(); + let r1 = A * (1.0 - e2) / (1.0 - e2 * sin1.powi(2)).powf(1.5); + let d = (easting - FALSE_EASTING) / (n1 * K0); + let phi = phi1 + - (n1 * tan1 / r1) + * (d.powi(2) / 2.0 + - (5.0 + 3.0 * t1 + 10.0 * c1 - 4.0 * c1.powi(2) - 9.0 * ep2) * d.powi(4) / 24.0 + + (61.0 + 90.0 * t1 + 298.0 * c1 + 45.0 * t1.powi(2) + - 252.0 * ep2 + - 3.0 * c1.powi(2)) + * d.powi(6) + / 720.0); + let lambda = (d - (1.0 + 2.0 * t1 + c1) * d.powi(3) / 6.0 + + (5.0 - 2.0 * c1 + 28.0 * t1 - 3.0 * c1.powi(2) + 8.0 * ep2 + 24.0 * t1.powi(2)) + * d.powi(5) + / 120.0) + / cos1; + (ZONE_47_MERIDIAN + lambda.to_degrees(), phi.to_degrees()) +} + +/// Even-odd test of `point` against the closed `ring`, both `(lon, lat)`. +pub fn inside(point: (f64, f64), ring: &[(f64, f64)]) -> bool { + let (x, y) = point; + let mut odd = false; + for (i, &(x1, y1)) in ring.iter().enumerate() { + let (x2, y2) = ring[(i + 1) % ring.len()]; + if (y1 > y) != (y2 > y) { + let crossing = x1 + (y - y1) * (x2 - x1) / (y2 - y1); + if x < crossing { + odd = !odd; + } + } + } + odd +} + +/// Metres from `point` to the nearest edge of `ring`, with lon/lat scaled +/// by the WGS 84 radii of curvature at the point's latitude. +pub fn edge_distance_m(point: (f64, f64), ring: &[(f64, f64)]) -> f64 { + let e2 = F * (2.0 - F); + let lat = point.1.to_radians(); + let w = (1.0 - e2 * lat.sin().powi(2)).sqrt(); + let meridional = A * (1.0 - e2) / w.powi(3); + let normal = A / w; + let kx = normal * lat.cos() * std::f64::consts::PI / 180.0; + let ky = meridional * std::f64::consts::PI / 180.0; + let scale = |(x, y): (f64, f64)| (x * kx, y * ky); + let (px, py) = scale(point); + let mut nearest = f64::INFINITY; + for (i, &a) in ring.iter().enumerate() { + let (ax, ay) = scale(a); + let (bx, by) = scale(ring[(i + 1) % ring.len()]); + let (dx, dy) = (bx - ax, by - ay); + let t = (((px - ax) * dx + (py - ay) * dy) / (dx * dx + dy * dy)).clamp(0.0, 1.0); + nearest = nearest.min((px - ax - t * dx).hypot(py - ay - t * dy)); + } + nearest +} + +#[test] +fn inverse_matches_a_known_point() { + // The zone origin row: easting 500000 lies on the central meridian. + let (lon, lat) = utm47n_to_lonlat(500_000.0, 0.0); + assert!( + (lon - 99.0).abs() < 1e-12 && lat.abs() < 1e-12, + "({lon}, {lat})" + ); + // The WGS 84 meridian arc from 0 to 1 degree north is 110574.389 m. + let (lon, lat) = utm47n_to_lonlat(500_000.0, 110_574.389 * K0); + assert!((lon - 99.0).abs() < 1e-12, "{lon}"); + assert!((lat - 1.0).abs() < 1e-7, "{lat}"); +} diff --git a/nodedb/tests/wire/cases/array_geotiff_e2e.rs b/nodedb/tests/wire/cases/array_geotiff_e2e.rs new file mode 100644 index 000000000..4e692e05c --- /dev/null +++ b/nodedb/tests/wire/cases/array_geotiff_e2e.rs @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! End to end: a GeoTIFF loads into a pixel-mode and a world-mode array, +//! and `ARRAY_AGG` over a WGS 84 polygon answers what an independent +//! computation answers. +//! +//! The answer is computed in the test: each pixel centre through the file +//! geotransform, a Snyder UTM inverse to WGS 84, an even-odd +//! point-in-polygon test, nodata excluded. Every pixel centre lies at +//! least 0.5 m from the polygon edges, so a millimetre difference between +//! two projection series cannot move a pixel across an edge. + +use crate::harness::TestServer; + +use super::array_geotiff::fixture::{ + HEIGHT, KEPT_PIXELS, WIDTH, in_nodata_block, pixel_centre, sample, scene_file, +}; +use super::array_geotiff::support::{ + PIXEL_GEOREF, SCENE_ATTRS, aggregate, copy_geotiff, create, pixel_array, world_array, +}; +use super::array_geotiff::utm::{edge_distance_m, inside, utm47n_to_lonlat}; + +/// An oblique quadrilateral inside the scene footprint, `(lon, lat)`, +/// counter-clockwise. Its edges run between pixel-centre rows, so every +/// centre keeps a margin. +const ZONE: [(f64, f64); 4] = [ + (99.913_085_804, 6.323_341_113), + (99.919_310_793, 6.291_911_559), + (99.956_934_156, 6.287_137_216), + (99.962_236_586, 6.313_413_463), +]; + +/// The least distance, in metres, from any pixel centre to a zone edge. +const MARGIN_M: f64 = 0.5; + +/// Relative tolerance of a mean. The engine can merge partial means in +/// any order, so the last bits can differ. +const MEAN_TOLERANCE: f64 = 1e-9; + +fn zone_ewkt() -> String { + let ring: Vec = ZONE + .iter() + .chain(ZONE.first()) + .map(|(lon, lat)| format!("{lon} {lat}")) + .collect(); + format!("SRID=4326;POLYGON(({}))", ring.join(", ")) +} + +/// The `ARRAY_AGG` tail of a zone query: no group, no slice, the zone, +/// and the GEOREF axes. +fn zone_tail() -> String { + format!(", NULL, NULL, '{}', NULL", zone_ewkt()) +} + +/// The independent answer for band `k`: the count and the sum of the +/// kept pixels whose centre lies inside the zone. +struct Expected { + count: u64, + sum: u64, + nodata_inside: u64, +} + +fn expected(k: u32) -> Expected { + let mut answer = Expected { + count: 0, + sum: 0, + nodata_inside: 0, + }; + for r in 0..HEIGHT { + for c in 0..WIDTH { + let (easting, northing) = pixel_centre(r, c); + let lonlat = utm47n_to_lonlat(easting, northing); + let margin = edge_distance_m(lonlat, &ZONE); + assert!( + margin >= MARGIN_M, + "pixel ({r}, {c}) centre is {margin} m from a zone edge" + ); + if !inside(lonlat, &ZONE) { + continue; + } + if in_nodata_block(r, c) { + answer.nodata_inside += 1; + continue; + } + answer.count += 1; + answer.sum += u64::from(sample(k, r, c)); + } + } + answer +} + +/// The zone answers of `array` for `b2`: count, sum, mean. +async fn zone_answers(server: &TestServer, array: &str) -> [f64; 3] { + let tail = zone_tail(); + [ + aggregate(server, array, "b2", "count", &tail).await, + aggregate(server, array, "b2", "sum", &tail).await, + aggregate(server, array, "b2", "mean", &tail).await, + ] +} + +fn assert_answers(array: &str, got: [f64; 3], want: &Expected) { + let [count, sum, mean] = got; + assert_eq!(count, want.count as f64, "{array} count"); + assert_eq!(sum, want.sum as f64, "{array} sum"); + let want_mean = want.sum as f64 / want.count as f64; + assert!( + (mean - want_mean).abs() <= MEAN_TOLERANCE * want_mean, + "{array} mean {mean} != {want_mean}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn zone_aggregates_match_an_independent_projection() { + let want = expected(2); + assert!( + want.count > 10_000, + "the zone covers a large part of the scene" + ); + assert_eq!( + want.nodata_inside, 100, + "the nodata block lies inside the zone, so the exclusion is tested" + ); + + let server = TestServer::start().await; + create( + &server, + &pixel_array("scene_px", "0..191", SCENE_ATTRS, PIXEL_GEOREF), + ) + .await; + create(&server, &world_array("scene_w")).await; + let (_file, path) = scene_file(); + for array in ["scene_px", "scene_w"] { + let loaded = copy_geotiff(&server, array, &path, "") + .await + .unwrap_or_else(|e| panic!("COPY {array}: {e:?}")); + assert_eq!(loaded, KEPT_PIXELS, "{array}: the tag is COPY 49052"); + } + + let pixel = zone_answers(&server, "scene_px").await; + let world = zone_answers(&server, "scene_w").await; + assert_answers("scene_px", pixel, &want); + assert_answers("scene_w", world, &want); + assert_eq!(pixel[..2], world[..2], "both arrays count and sum the same"); + assert!( + (pixel[2] - world[2]).abs() <= MEAN_TOLERANCE * pixel[2], + "both arrays average the same: {} vs {}", + pixel[2], + world[2] + ); +} diff --git a/nodedb/tests/wire/cases/array_geotiff_load.rs b/nodedb/tests/wire/cases/array_geotiff_load.rs new file mode 100644 index 000000000..bc9f05f60 --- /dev/null +++ b/nodedb/tests/wire/cases/array_geotiff_load.rs @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `COPY FROM '.tif'`: what a GeoTIFF load writes. +//! +//! The scene holds 49152 pixels, so its cells span several 4 MiB load +//! chunks. The all-zero block is nodata under `GDAL_NODATA` and loads no +//! cell. `NODATA` replaces the file value, `BANDS` picks the bands, and +//! `TIME` fills a TIMESTAMP dim. + +use crate::harness::TestServer; + +use super::array_geotiff::fixture::{ + HEIGHT, KEPT_PIXELS, NODATA_COLS, NODATA_ROWS, WIDTH, in_nodata_block, sample, scene_file, +}; +use super::array_geotiff::support::{ + PIXEL_GEOREF, SCENE_ATTRS, aggregate, copy_geotiff, create, pixel_array, pixel_cells, +}; + +/// The sum of band `k` over every pixel `keep` accepts. +fn band_sum(k: u32, keep: impl Fn(u32, u32) -> bool) -> f64 { + let mut sum = 0u64; + for r in 0..HEIGHT { + for c in 0..WIDTH { + if keep(r, c) { + sum += u64::from(sample(k, r, c)); + } + } + } + sum as f64 +} + +async fn load_scene(server: &TestServer, name: &str, path: &str) { + create( + server, + &pixel_array(name, "0..191", SCENE_ATTRS, PIXEL_GEOREF), + ) + .await; + let loaded = copy_geotiff(server, name, path, "") + .await + .unwrap_or_else(|e| panic!("COPY {name}: {e:?}")); + assert_eq!(loaded, KEPT_PIXELS, "the tag is COPY 49052"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_multi_chunk_load_keeps_every_pixel_but_nodata() { + let server = TestServer::start().await; + let (_file, path) = scene_file(); + load_scene(&server, "geo_load", &path).await; + + assert_eq!( + aggregate(&server, "geo_load", "b1", "count", "").await, + KEPT_PIXELS as f64 + ); + let want = band_sum(3, |r, c| !in_nodata_block(r, c)); + assert_eq!(aggregate(&server, "geo_load", "b3", "sum", "").await, want); + + // A ring of one pixel around the nodata block holds cells, the block none. + let (r0, r1) = (NODATA_ROWS.start - 1, NODATA_ROWS.end); + let (c0, c1) = (NODATA_COLS.start - 1, NODATA_COLS.end); + let cells = pixel_cells(&server, "geo_load", (r0, r1), (c0, c1)).await; + assert_eq!( + cells.len(), + 12 * 12 - 100, + "the ring loads, the block does not" + ); + assert!(cells.keys().all(|&(r, c)| !in_nodata_block(r, c))); + let corner = cells.get(&(r0, c0)).expect("ring corner"); + assert_eq!(corner, &sample(1, r0, c0).to_string()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn bands_and_nodata_replace_the_file_defaults() { + let server = TestServer::start().await; + let (_file, path) = scene_file(); + create( + &server, + &pixel_array("geo_b1", "0..191", "b1 UINT16", PIXEL_GEOREF), + ) + .await; + // NODATA 1000 replaces GDAL_NODATA 0, so the zero block loads and every + // b1 of 1000 is skipped. Bands 2 and 3 load nowhere. + let loaded = copy_geotiff( + &server, + "geo_b1", + &path, + ", BANDS (b1, SKIP, SKIP), NODATA 1000", + ) + .await + .unwrap_or_else(|e| panic!("COPY geo_b1: {e:?}")); + let skipped = (0..HEIGHT) + .flat_map(|r| (0..WIDTH).map(move |c| (r, c))) + .filter(|&(r, c)| sample(1, r, c) == 1000) + .count() as u64; + assert!(skipped > 0, "some b1 samples are 1000"); + assert_eq!(loaded, u64::from(WIDTH * HEIGHT) - skipped); + + let zero = pixel_cells(&server, "geo_b1", (100, 100), (120, 120)).await; + assert_eq!(zero.get(&(100, 120)).map(String::as_str), Some("0")); + let origin = pixel_cells(&server, "geo_b1", (0, 0), (0, 0)).await; + assert!(origin.is_empty(), "b1(0, 0) is 1000, the new nodata"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_nodata_band_in_a_not_null_attr_names_its_pixel() { + let server = TestServer::start().await; + let (_file, path) = scene_file(); + create( + &server, + &pixel_array( + "geo_strict", + "0..191", + "b1 UINT16 NOT NULL, b2 UINT16, b3 UINT16", + PIXEL_GEOREF, + ), + ) + .await; + // Pixel (0, 0) holds 1000, 2000, 3000: only b1 is nodata, so it loads. + let err = copy_geotiff(&server, "geo_strict", &path, ", NODATA 1000") + .await + .expect_err("b1 is NOT NULL"); + assert_eq!(err.code, "23502", "{err:?}"); + assert!(err.message.contains("row 1:"), "{err:?}"); + assert!(err.message.contains("(pixel row 0, col 0)"), "{err:?}"); + assert!(err.message.ends_with("(0 rows committed)"), "{err:?}"); + assert_eq!( + aggregate(&server, "geo_strict", "b2", "count", "").await, + 0.0 + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn time_fills_the_timestamp_dim() { + let server = TestServer::start().await; + let (_file, path) = scene_file(); + create( + &server, + &format!( + "CREATE ARRAY geo_time \ + DIMS (t TIMESTAMP_MS [1735689600000..1798761600000], \ + row INT64 [0..191], col INT64 [0..255]) \ + ATTRS ({SCENE_ATTRS}) TILE_EXTENTS (86400000, 64, 64) {PIXEL_GEOREF}" + ), + ) + .await; + let loaded = copy_geotiff(&server, "geo_time", &path, ", TIME '2025-06-01T00:00:00Z'") + .await + .unwrap_or_else(|e| panic!("COPY geo_time: {e:?}")); + assert_eq!(loaded, KEPT_PIXELS); + let rows = server + .query_named_rows("SELECT * FROM ARRAY_AGG('geo_time', 'b1', 'count', 't')") + .await + .expect("count by t"); + assert_eq!(rows.len(), 1, "one scene instant: {rows:?}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_load_inside_a_block_rolls_back_whole() { + let server = TestServer::start().await; + let (_file, path) = scene_file(); + create( + &server, + &pixel_array("geo_txn", "0..191", SCENE_ATTRS, PIXEL_GEOREF), + ) + .await; + server.exec("BEGIN").await.expect("BEGIN"); + let loaded = copy_geotiff(&server, "geo_txn", &path, "") + .await + .unwrap_or_else(|e| panic!("COPY in a block: {e:?}")); + assert_eq!(loaded, KEPT_PIXELS); + server.exec("ROLLBACK").await.expect("ROLLBACK"); + assert_eq!( + aggregate(&server, "geo_txn", "b1", "count", "").await, + 0.0, + "ROLLBACK discards every staged chunk" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_loaded_scene_survives_restart() { + let server = TestServer::start().await; + let (_file, path) = scene_file(); + load_scene(&server, "geo_restart", &path).await; + + let (server, dir) = server.take_dir(); + server.graceful_shutdown().await; + let (server, _dir) = TestServer::open_on_path(dir).await; + + assert_eq!( + aggregate(&server, "geo_restart", "b1", "count", "").await, + KEPT_PIXELS as f64 + ); + let want = band_sum(2, |r, c| !in_nodata_block(r, c)); + assert_eq!( + aggregate(&server, "geo_restart", "b2", "sum", "").await, + want + ); +} diff --git a/nodedb/tests/wire/cases/array_geotiff_refusals.rs b/nodedb/tests/wire/cases/array_geotiff_refusals.rs new file mode 100644 index 000000000..7e446c702 --- /dev/null +++ b/nodedb/tests/wire/cases/array_geotiff_refusals.rs @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `COPY FROM '.tif'` refusals. Every open-time check runs +//! before the first block decodes, so a refused load commits no row. + +use crate::harness::TestServer; + +use super::array_geotiff::fixture::{scene_file, scene_tiff, write_tiff}; +use super::array_geotiff::support::{ + PIXEL_GEOREF, SCENE_ATTRS, assert_refused, create, pixel_array, +}; + +/// One refused array: its name, DDL, COPY options, and SQLSTATE. +struct Case { + name: &'static str, + ddl: String, + extra: &'static str, + code: &'static str, +} + +fn cases() -> Vec { + let case = |name, ddl, extra, code| Case { + name, + ddl, + extra, + code, + }; + vec![ + // u16 samples do not fit UINT8. + case( + "geo_narrow", + pixel_array( + "geo_narrow", + "0..191", + "b1 UINT8, b2 UINT16, b3 UINT16", + PIXEL_GEOREF, + ), + "", + "42804", + ), + // The file is EPSG:32647. + case( + "geo_srid", + pixel_array( + "geo_srid", + "0..191", + SCENE_ATTRS, + "GEOREF (SRID 32648, X col, Y row, TRANSFORM (600000, 30, 0, 700000, 0, -30))", + ), + "", + "22023", + ), + // Half a pixel east of the file grid. + case( + "geo_shift", + pixel_array( + "geo_shift", + "0..191", + SCENE_ATTRS, + "GEOREF (SRID 32647, X col, Y row, TRANSFORM (600015, 30, 0, 700000, 0, -30))", + ), + "", + "22023", + ), + // 192 rows do not fit [0..99]. + case( + "geo_small", + pixel_array("geo_small", "0..99", SCENE_ATTRS, PIXEL_GEOREF), + "", + "22003", + ), + // No GEOREF places no pixel. + case( + "geo_plain", + pixel_array("geo_plain", "0..191", SCENE_ATTRS, ""), + "", + "55000", + ), + // No band fills a NOT NULL attr. + case( + "geo_qa", + pixel_array( + "geo_qa", + "0..191", + "b1 UINT16, b2 UINT16, b3 UINT16, qa UINT8 NOT NULL", + PIXEL_GEOREF, + ), + "", + "23502", + ), + // A TIMESTAMP dim needs TIME. + case( + "geo_untimed", + format!( + "CREATE ARRAY geo_untimed \ + DIMS (t TIMESTAMP_MS [1735689600000..1798761600000], \ + row INT64 [0..191], col INT64 [0..255]) \ + ATTRS ({SCENE_ATTRS}) TILE_EXTENTS (86400000, 64, 64) {PIXEL_GEOREF}" + ), + "", + "42601", + ), + // A BANDS list must name one entry per file band. + case( + "geo_bands", + pixel_array("geo_bands", "0..191", SCENE_ATTRS, PIXEL_GEOREF), + ", BANDS (b1, b2)", + "22023", + ), + ] +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn open_time_refusals_commit_no_row() { + let server = TestServer::start().await; + let (_file, path) = scene_file(); + for case in cases() { + create(&server, &case.ddl).await; + assert_refused(&server, case.name, &path, case.extra, case.code).await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_corrupt_file_is_a_bad_copy_file_format() { + let server = TestServer::start().await; + create( + &server, + &pixel_array("geo_corrupt", "0..191", SCENE_ATTRS, PIXEL_GEOREF), + ) + .await; + // The encoder writes the tiles, then the directory, so a cut file + // keeps its header and loses the directory it points to. + let bytes = scene_tiff(); + let cut = write_tiff(&bytes[..bytes.len() / 2]); + let path = cut.path().to_string_lossy().into_owned(); + assert_refused(&server, "geo_corrupt", &path, "", "22P04").await; +} diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index df7f7622a..fce34324f 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -10,6 +10,12 @@ mod aggregate_routing_multicore; mod array_agg_group_types; mod array_copy_from; mod array_dml_in_bodies; +mod array_georef; +mod array_georef_zone; +mod array_geotiff; +mod array_geotiff_e2e; +mod array_geotiff_load; +mod array_geotiff_refusals; mod array_narrow_attrs; mod backup_support; mod bitemporal_array_sql; diff --git a/nodedb/tests/wire/cases/sql_path_jail.rs b/nodedb/tests/wire/cases/sql_path_jail.rs index 8b2d0a7d7..475a87da9 100644 --- a/nodedb/tests/wire/cases/sql_path_jail.rs +++ b/nodedb/tests/wire/cases/sql_path_jail.rs @@ -12,8 +12,9 @@ //! //! `COPY FROM ''` is a real feature, but its path jail //! still blocks (a) non-absolute paths and (b) any path containing a -//! `..` component. Absolute, non-traversal paths are accepted under -//! the server UID's filesystem permissions. +//! `..` component. A server-path `COPY FROM` or `COPY TO` needs +//! superuser (`42501`), and `COPY FROM` reads regular files only +//! (`58030` for a FIFO, device, or directory). use crate::harness::TestServer; @@ -210,12 +211,179 @@ async fn copy_from_rejects_path_form_absolute_system_file() { assert_structural_rejection(&msg, &["stdin", "copy"]); } -// `COPY FROM ''` is a real feature — -// see `control/server/pgwire/ddl/collection/copy_from/`. The path -// jail is "no relative paths, no `..` traversal", not "no paths at -// all". An absolute, non-traversal path that resolves to a readable -// file under the server's UID is accepted; it's the operator's job -// to grant or deny that UID access to the path. +// `COPY FROM ''` is a real feature. The +// path jail refuses relative paths and `..` traversal. Beyond the jail, +// a server path needs superuser, and it must open as a regular file. +// The tests below lock both rules, for COPY FROM and COPY TO. + +const COPY_PASSWORD: &str = "pjail-copy-secret-3"; + +/// Bound on a COPY against a FIFO or device: a hang fails the test. +const COPY_DEADLINE: std::time::Duration = std::time::Duration::from_secs(20); + +/// Create `user` with the default ReadWrite role and run `sql` as that +/// user. `Err((sqlstate, message))` on refusal. +async fn run_as_writer(server: &TestServer, user: &str, sql: &str) -> Result<(), (String, String)> { + server + .exec(&format!("CREATE USER {user} PASSWORD '{COPY_PASSWORD}'")) + .await + .unwrap_or_else(|e| panic!("create user {user}: {e}")); + let (client, handle) = server + .connect_as(user, COPY_PASSWORD) + .await + .unwrap_or_else(|e| panic!("connect as {user}: {e}")); + let result = client.simple_query(sql).await; + drop(client); + handle.abort(); + result.map(|_| ()).map_err(|e| { + e.as_db_error().map_or_else( + || (String::new(), e.to_string()), + |db| (db.code().code().to_string(), db.message().to_string()), + ) + }) +} + +/// Run `sql` as the superuser under [`COPY_DEADLINE`]; the lower-cased +/// `sqlstate: message` of its refusal. +async fn superuser_refusal(server: &TestServer, sql: &str) -> String { + tokio::time::timeout(COPY_DEADLINE, err_of(server, sql)) + .await + .unwrap_or_else(|_| panic!("COPY hung on a non-regular file: {sql}")) +} + +#[tokio::test] +async fn copy_from_server_path_requires_superuser() { + let server = TestServer::start().await; + server + .exec( + "CREATE COLLECTION pj_su (id TEXT PRIMARY KEY, v TEXT) WITH (engine='document_strict')", + ) + .await + .expect("create collection"); + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("rows.ndjson"); + std::fs::write(&path, "{\"id\":\"a\",\"v\":\"x\"}\n").expect("write rows"); + + let refusal = run_as_writer( + &server, + "pj_writer_from", + &format!("COPY pj_su FROM '{}'", path.display()), + ) + .await + .expect_err("a non-superuser server-path COPY FROM must be refused"); + assert_eq!(refusal.0, "42501", "{refusal:?}"); + assert!(refusal.1.contains("superuser"), "{refusal:?}"); + let rows = server + .query_text("SELECT id FROM pj_su") + .await + .expect("scan"); + assert!(rows.is_empty(), "a refused COPY wrote rows: {rows:?}"); + + // The superuser runs the same statement. + server + .exec(&format!("COPY pj_su FROM '{}'", path.display())) + .await + .expect("superuser COPY FROM"); +} + +#[tokio::test] +async fn copy_to_server_path_requires_superuser() { + let server = TestServer::start().await; + server + .exec("CREATE COLLECTION pj_su_to (id TEXT PRIMARY KEY, v TEXT) WITH (engine='document_strict')") + .await + .expect("create collection"); + server + .exec("INSERT INTO pj_su_to (id, v) VALUES ('a', 'x')") + .await + .expect("seed"); + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("out.ndjson"); + + let refusal = run_as_writer( + &server, + "pj_writer_to", + &format!("COPY pj_su_to TO '{}'", path.display()), + ) + .await + .expect_err("a non-superuser server-path COPY TO must be refused"); + assert_eq!(refusal.0, "42501", "{refusal:?}"); + assert!(!path.exists(), "a refused COPY TO wrote {}", path.display()); +} + +#[tokio::test] +async fn copy_from_a_device_is_refused() { + let server = TestServer::start().await; + server + .exec("CREATE COLLECTION pj_dev (id TEXT PRIMARY KEY, v TEXT) WITH (engine='document_strict')") + .await + .expect("create collection"); + let msg = superuser_refusal(&server, "COPY pj_dev FROM '/dev/zero' WITH (FORMAT csv)").await; + assert!(msg.starts_with("58030"), "{msg}"); + assert!(msg.contains("not a regular file"), "{msg}"); +} + +#[tokio::test] +async fn copy_from_a_symlink_to_a_fifo_or_a_directory_is_refused() { + let server = TestServer::start().await; + server + .exec("CREATE COLLECTION pj_fifo (id TEXT PRIMARY KEY, v TEXT) WITH (engine='document_strict')") + .await + .expect("create collection"); + let dir = tempfile::tempdir().expect("tempdir"); + let fifo = dir.path().join("pipe"); + let status = std::process::Command::new("mkfifo") + .arg(&fifo) + .status() + .expect("run mkfifo"); + assert!(status.success(), "mkfifo failed: {status}"); + let link = dir.path().join("rows.csv"); + std::os::unix::fs::symlink(&fifo, &link).expect("symlink to fifo"); + + let msg = superuser_refusal(&server, &format!("COPY pj_fifo FROM '{}'", link.display())).await; + assert!(msg.starts_with("58030"), "{msg}"); + assert!(msg.contains("not a regular file"), "{msg}"); + + let subdir = dir.path().join("dir.csv"); + std::fs::create_dir(&subdir).expect("create dir"); + let msg = superuser_refusal( + &server, + &format!("COPY pj_fifo FROM '{}'", subdir.display()), + ) + .await; + assert!(msg.starts_with("58030"), "{msg}"); + assert!(msg.contains("not a regular file"), "{msg}"); +} + +#[tokio::test] +async fn geotiff_over_stdin_names_the_server_path_form() { + let server = TestServer::start().await; + server + .exec( + "CREATE ARRAY pj_scene DIMS (y INT64 [0..7], x INT64 [0..7]) \ + ATTRS (b1 FLOAT64) TILE_EXTENTS (4, 4)", + ) + .await + .expect("create array"); + let msg = superuser_refusal(&server, "COPY pj_scene FROM STDIN WITH (FORMAT geotiff)").await; + assert!(msg.starts_with("0a000"), "{msg}"); + assert!(msg.contains("from '.tif'"), "{msg}"); +} + +#[tokio::test] +async fn array_copy_from_a_device_is_refused() { + let server = TestServer::start().await; + server + .exec( + "CREATE ARRAY pj_arr DIMS (y INT64 [0..7], x INT64 [0..7]) \ + ATTRS (b1 FLOAT64) TILE_EXTENTS (4, 4)", + ) + .await + .expect("create array"); + let msg = superuser_refusal(&server, "COPY pj_arr FROM '/dev/zero' WITH (FORMAT csv)").await; + assert!(msg.starts_with("58030"), "{msg}"); + assert!(msg.contains("not a regular file"), "{msg}"); +} // ──────────────────────────────────────────────────────────────────── // EXPORT AUDIT LOG TO '' — same removal.